diff --git a/ppo_dir_v2/v0-20260524-184726/README.md b/ppo_dir_v2/v0-20260524-184726/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c99feef3e033bcc4c8ce2a71dc8267cebb8b7456 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/README.md @@ -0,0 +1,66 @@ +--- +library_name: transformers +model_name: v0-20260524-184726 +tags: +- generated_from_trainer +- ppo +- trl +licence: license +--- + +# Model Card for v0-20260524-184726 + +This model is a fine-tuned version of [None](https://huggingface.co/None). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + +This model was trained with PPO, a method introduced in [Fine-Tuning Language Models from Human Preferences](https://huggingface.co/papers/1909.08593). + +### Framework versions + +- TRL: 0.19.1 +- Transformers: 4.55.4 +- Pytorch: 2.7.1 +- Datasets: 3.3.2 +- Tokenizers: 0.21.4 + +## Citations + +Cite PPO as: + +```bibtex +@article{mziegler2019fine-tuning, + title = {{Fine-Tuning Language Models from Human Preferences}}, + author = {Daniel M. Ziegler and Nisan Stiennon and Jeffrey Wu and Tom B. Brown and Alec Radford and Dario Amodei and Paul F. Christiano and Geoffrey Irving}, + year = 2019, + eprint = {arXiv:1909.08593} +} +``` + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/args.json b/ppo_dir_v2/v0-20260524-184726/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2b7330636e488c8d17319fc78addd266d3d92e9 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c56d56bc3e743ef74af3a9cf3846cf682d30c6b4dfbd0612ecf57b2f82d4419c +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bc2784f9f0d8f590651a8b483c7f400af234c69b --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/trainer_state.json @@ -0,0 +1,21035 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 16000, + "epoch": 0.8080808080808081, + "eval_steps": 100, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + }, + { + "episode": 11216, + "epoch": 0.5664646464646464, + "eps": 0, + "loss/policy_avg": -0.09055965393781662, + "loss/value_avg": 0.5455071926116943, + "lr": 4.334128543718295e-06, + "objective/entropy": -74.95745849609375, + "objective/kl": 33.42132568359375, + "objective/non_score_reward": -1.6710662841796875, + "objective/rlhf_reward": 14.828933715820312, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0013184514828026295, + "policy/clipfrac_avg": 0.00809475127607584, + "policy/entropy_avg": 0.17902374267578125, + "step": 701, + "val/clipfrac_avg": 0.18833743035793304, + "val/num_eos_tokens": 358, + "val/ratio": 0.9879262447357178, + "val/ratio_var": 6.120221951277927e-05 + }, + { + "episode": 11232, + "epoch": 0.5672727272727273, + "eps": 0, + "loss/policy_avg": 0.2261451780796051, + "loss/value_avg": 1.5403046607971191, + "lr": 4.320892800523827e-06, + "objective/entropy": -90.61648559570312, + "objective/kl": 27.537485122680664, + "objective/non_score_reward": -1.3768744468688965, + "objective/rlhf_reward": 12.826250076293945, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009536159923300147, + "policy/clipfrac_avg": 0.01545755285769701, + "policy/entropy_avg": 0.20562219619750977, + "step": 702, + "val/clipfrac_avg": 0.033450622111558914, + "val/num_eos_tokens": 595, + "val/ratio": 0.9873440265655518, + "val/ratio_var": 7.613677007611841e-05 + }, + { + "episode": 11248, + "epoch": 0.5680808080808081, + "eps": 0, + "loss/policy_avg": -0.05153674632310867, + "loss/value_avg": 0.18954747915267944, + "lr": 4.307661903773129e-06, + "objective/entropy": -56.119659423828125, + "objective/kl": 43.216548919677734, + "objective/non_score_reward": -2.160827398300171, + "objective/rlhf_reward": 15.80792236328125, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010682737920433283, + "policy/clipfrac_avg": 0.005114596337080002, + "policy/entropy_avg": 0.20781707763671875, + "step": 703, + "val/clipfrac_avg": 0.12178023159503937, + "val/num_eos_tokens": 244, + "val/ratio": 0.9860707521438599, + "val/ratio_var": 5.392924140323885e-05 + }, + { + "episode": 11264, + "epoch": 0.5688888888888889, + "eps": 0, + "loss/policy_avg": 0.22105136513710022, + "loss/value_avg": 0.43452760577201843, + "lr": 4.294435947888405e-06, + "objective/entropy": -180.00924682617188, + "objective/kl": 32.27890396118164, + "objective/non_score_reward": -1.6139451265335083, + "objective/rlhf_reward": 12.425117492675781, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007389815291389823, + "policy/clipfrac_avg": 0.00613048393279314, + "policy/entropy_avg": 0.15887287259101868, + "step": 704, + "val/clipfrac_avg": 0.12273523956537247, + "val/num_eos_tokens": 611, + "val/ratio": 0.9900252819061279, + "val/ratio_var": 6.185931124491617e-05 + }, + { + "episode": 11280, + "epoch": 0.5696969696969697, + "eps": 0, + "loss/policy_avg": -0.044020622968673706, + "loss/value_avg": 1.459969401359558, + "lr": 4.281215027256592e-06, + "objective/entropy": -73.2503890991211, + "objective/kl": 29.207317352294922, + "objective/non_score_reward": -1.460365891456604, + "objective/rlhf_reward": 15.766195297241211, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010228066239506006, + "policy/clipfrac_avg": 0.010731351561844349, + "policy/entropy_avg": 0.17015647888183594, + "step": 705, + "val/clipfrac_avg": 0.1418685019016266, + "val/num_eos_tokens": 378, + "val/ratio": 0.9865360856056213, + "val/ratio_var": 0.0001155980207840912 + }, + { + "episode": 11296, + "epoch": 0.5705050505050505, + "eps": 0, + "loss/policy_avg": 0.10734500735998154, + "loss/value_avg": 0.40052735805511475, + "lr": 4.2679992362287e-06, + "objective/entropy": -91.05632019042969, + "objective/kl": 39.329261779785156, + "objective/non_score_reward": -1.966463327407837, + "objective/rlhf_reward": 12.744474411010742, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010600072564557195, + "policy/clipfrac_avg": 0.005786027293652296, + "policy/entropy_avg": 0.23768234252929688, + "step": 706, + "val/clipfrac_avg": 0.032500892877578735, + "val/num_eos_tokens": 203, + "val/ratio": 0.9860942959785461, + "val/ratio_var": 5.682241317117587e-05 + }, + { + "episode": 11312, + "epoch": 0.5713131313131313, + "eps": 0, + "loss/policy_avg": -0.5062837600708008, + "loss/value_avg": 0.7742470502853394, + "lr": 4.254788669119127e-06, + "objective/entropy": -93.2535400390625, + "objective/kl": 34.540130615234375, + "objective/non_score_reward": -1.727006435394287, + "objective/rlhf_reward": 14.788618087768555, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008475214708596468, + "policy/clipfrac_avg": 0.027025574818253517, + "policy/entropy_avg": 0.21597671508789062, + "step": 707, + "val/clipfrac_avg": 0.1775030940771103, + "val/num_eos_tokens": 165, + "val/ratio": 0.9894334077835083, + "val/ratio_var": 9.222733206115663e-05 + }, + { + "episode": 11328, + "epoch": 0.5721212121212121, + "eps": 0, + "loss/policy_avg": -0.1463669091463089, + "loss/value_avg": 0.5632686614990234, + "lr": 4.241583420204998e-06, + "objective/entropy": -110.37503051757812, + "objective/kl": 36.62909698486328, + "objective/non_score_reward": -1.831454873085022, + "objective/rlhf_reward": 12.89510726928711, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009251610608771443, + "policy/clipfrac_avg": 0.00941617414355278, + "policy/entropy_avg": 0.16801166534423828, + "step": 708, + "val/clipfrac_avg": 0.04514690116047859, + "val/num_eos_tokens": 267, + "val/ratio": 0.9897239208221436, + "val/ratio_var": 7.692311191931367e-05 + }, + { + "episode": 11344, + "epoch": 0.572929292929293, + "eps": 0, + "loss/policy_avg": -0.35295569896698, + "loss/value_avg": 0.7035583257675171, + "lr": 4.228383583725471e-06, + "objective/entropy": -76.27204132080078, + "objective/kl": 31.900005340576172, + "objective/non_score_reward": -1.5950002670288086, + "objective/rlhf_reward": 14.053436279296875, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010020419722422957, + "policy/clipfrac_avg": 0.011507621966302395, + "policy/entropy_avg": 0.16324424743652344, + "step": 709, + "val/clipfrac_avg": 0.12508229911327362, + "val/num_eos_tokens": 173, + "val/ratio": 0.9896982908248901, + "val/ratio_var": 4.9723090342013165e-05 + }, + { + "episode": 11360, + "epoch": 0.5737373737373738, + "eps": 0, + "loss/policy_avg": 0.160404771566391, + "loss/value_avg": 1.1914963722229004, + "lr": 4.215189253881093e-06, + "objective/entropy": -79.55085754394531, + "objective/kl": 36.81852722167969, + "objective/non_score_reward": -1.8409266471862793, + "objective/rlhf_reward": 12.159073829650879, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0011080782860517502, + "policy/clipfrac_avg": 0.006503967102617025, + "policy/entropy_avg": 0.19612884521484375, + "step": 710, + "val/clipfrac_avg": 0.08140315115451813, + "val/num_eos_tokens": 538, + "val/ratio": 0.9866147041320801, + "val/ratio_var": 7.869482942624018e-05 + }, + { + "episode": 11376, + "epoch": 0.5745454545454546, + "eps": 0, + "loss/policy_avg": 0.07548795640468597, + "loss/value_avg": 0.5138013362884521, + "lr": 4.2020005248331056e-06, + "objective/entropy": -119.03164672851562, + "objective/kl": 34.31198501586914, + "objective/non_score_reward": -1.7155992984771729, + "objective/rlhf_reward": 13.503150939941406, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007362160831689835, + "policy/clipfrac_avg": 0.00589254405349493, + "policy/entropy_avg": 0.14674091339111328, + "step": 711, + "val/clipfrac_avg": 0.02668866142630577, + "val/num_eos_tokens": 515, + "val/ratio": 0.990146279335022, + "val/ratio_var": 3.9673428545938805e-05 + }, + { + "episode": 11392, + "epoch": 0.5753535353535354, + "eps": 0, + "loss/policy_avg": -0.2925986051559448, + "loss/value_avg": 0.3821415603160858, + "lr": 4.1888174907027775e-06, + "objective/entropy": -112.06816101074219, + "objective/kl": 38.006500244140625, + "objective/non_score_reward": -1.900325059890747, + "objective/rlhf_reward": 13.990300178527832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.000874843040946871, + "policy/clipfrac_avg": 0.007544378750026226, + "policy/entropy_avg": 0.16875195503234863, + "step": 712, + "val/clipfrac_avg": 0.025153353810310364, + "val/num_eos_tokens": 4, + "val/ratio": 0.9893897771835327, + "val/ratio_var": 8.519255788996816e-05 + }, + { + "episode": 11408, + "epoch": 0.5761616161616162, + "eps": 0, + "loss/policy_avg": -0.5444605350494385, + "loss/value_avg": 1.517474889755249, + "lr": 4.175640245570742e-06, + "objective/entropy": -92.28114318847656, + "objective/kl": 35.69511413574219, + "objective/non_score_reward": -1.7847557067871094, + "objective/rlhf_reward": 13.73868179321289, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008441185345873237, + "policy/clipfrac_avg": 0.005670716054737568, + "policy/entropy_avg": 0.1591033935546875, + "step": 713, + "val/clipfrac_avg": 0.1791025698184967, + "val/num_eos_tokens": 98, + "val/ratio": 0.9903852939605713, + "val/ratio_var": 4.043114677187987e-05 + }, + { + "episode": 11424, + "epoch": 0.576969696969697, + "eps": 0, + "loss/policy_avg": -0.3059786260128021, + "loss/value_avg": 0.6795393228530884, + "lr": 4.162468883476319e-06, + "objective/entropy": -193.77468872070312, + "objective/kl": 42.17440414428711, + "objective/non_score_reward": -2.108720302581787, + "objective/rlhf_reward": 12.281905174255371, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008298080647364259, + "policy/clipfrac_avg": 0.00635810662060976, + "policy/entropy_avg": 0.15823698043823242, + "step": 714, + "val/clipfrac_avg": 0.0064142923802137375, + "val/num_eos_tokens": 1666, + "val/ratio": 0.9892975091934204, + "val/ratio_var": 6.228621350601315e-05 + }, + { + "episode": 11440, + "epoch": 0.5777777777777777, + "eps": 0, + "loss/policy_avg": -0.023257479071617126, + "loss/value_avg": 0.18199321627616882, + "lr": 4.149303498416838e-06, + "objective/entropy": -22.63348388671875, + "objective/kl": 44.308345794677734, + "objective/non_score_reward": -2.2154173851013184, + "objective/rlhf_reward": 14.82364559173584, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0012546201469376683, + "policy/clipfrac_avg": 0.007735411170870066, + "policy/entropy_avg": 0.240936279296875, + "step": 715, + "val/clipfrac_avg": 0.0028473599813878536, + "val/num_eos_tokens": 302, + "val/ratio": 0.9840776920318604, + "val/ratio_var": 5.949061960563995e-05 + }, + { + "episode": 11456, + "epoch": 0.5785858585858585, + "eps": 0, + "loss/policy_avg": 0.3237575888633728, + "loss/value_avg": 0.38197559118270874, + "lr": 4.136144184346979e-06, + "objective/entropy": -168.4746856689453, + "objective/kl": 40.63113784790039, + "objective/non_score_reward": -2.031557083129883, + "objective/rlhf_reward": 13.070005416870117, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0007934231543913484, + "policy/clipfrac_avg": 0.012050900608301163, + "policy/entropy_avg": 0.14413070678710938, + "step": 716, + "val/clipfrac_avg": 0.09322839975357056, + "val/num_eos_tokens": 1174, + "val/ratio": 0.9900047779083252, + "val/ratio_var": 5.29003118572291e-05 + }, + { + "episode": 11472, + "epoch": 0.5793939393939394, + "eps": 0, + "loss/policy_avg": 0.23654325306415558, + "loss/value_avg": 1.166879653930664, + "lr": 4.122991035178093e-06, + "objective/entropy": -96.77055358886719, + "objective/kl": 42.17526626586914, + "objective/non_score_reward": -2.1087632179260254, + "objective/rlhf_reward": 14.500611305236816, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000815618084743619, + "policy/clipfrac_avg": 0.008080050349235535, + "policy/entropy_avg": 0.14806270599365234, + "step": 717, + "val/clipfrac_avg": 0.02973807230591774, + "val/num_eos_tokens": 297, + "val/ratio": 0.9899397492408752, + "val/ratio_var": 3.871042645187117e-05 + }, + { + "episode": 11488, + "epoch": 0.5802020202020202, + "eps": 0, + "loss/policy_avg": -0.11968575417995453, + "loss/value_avg": 0.9426589608192444, + "lr": 4.109844144777537e-06, + "objective/entropy": -58.205047607421875, + "objective/kl": 42.6419792175293, + "objective/non_score_reward": -2.132099151611328, + "objective/rlhf_reward": 12.203838348388672, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009566791704855859, + "policy/clipfrac_avg": 0.0025268401950597763, + "policy/entropy_avg": 0.18893003463745117, + "step": 718, + "val/clipfrac_avg": 0.1098824143409729, + "val/num_eos_tokens": 16, + "val/ratio": 0.9869214296340942, + "val/ratio_var": 8.384165266761556e-05 + }, + { + "episode": 11504, + "epoch": 0.581010101010101, + "eps": 0, + "loss/policy_avg": 0.0208362378180027, + "loss/value_avg": 0.8499202728271484, + "lr": 4.096703606968007e-06, + "objective/entropy": -105.71292114257812, + "objective/kl": 40.70527648925781, + "objective/non_score_reward": -2.035263776779175, + "objective/rlhf_reward": 13.605360984802246, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008132618968375027, + "policy/clipfrac_avg": 0.004117591306567192, + "policy/entropy_avg": 0.15980243682861328, + "step": 719, + "val/clipfrac_avg": 0.04368842393159866, + "val/num_eos_tokens": 428, + "val/ratio": 0.9895676374435425, + "val/ratio_var": 6.342094275169075e-05 + }, + { + "episode": 11520, + "epoch": 0.5818181818181818, + "eps": 0, + "loss/policy_avg": -0.11627452820539474, + "loss/value_avg": 0.7368168830871582, + "lr": 4.083569515526853e-06, + "objective/entropy": -100.23733520507812, + "objective/kl": 32.93721008300781, + "objective/non_score_reward": -1.6468604803085327, + "objective/rlhf_reward": 15.134389877319336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008385346736758947, + "policy/clipfrac_avg": 0.009182492271065712, + "policy/entropy_avg": 0.17890548706054688, + "step": 720, + "val/clipfrac_avg": 0.06867726147174835, + "val/num_eos_tokens": 144, + "val/ratio": 0.9896779656410217, + "val/ratio_var": 7.192800694610924e-05 + }, + { + "episode": 11536, + "epoch": 0.5826262626262626, + "eps": 0, + "loss/policy_avg": -0.06581735610961914, + "loss/value_avg": 0.7933100461959839, + "lr": 4.070441964185428e-06, + "objective/entropy": -103.3985366821289, + "objective/kl": 39.985321044921875, + "objective/non_score_reward": -1.9992659091949463, + "objective/rlhf_reward": 14.547609329223633, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000881620217114687, + "policy/clipfrac_avg": 0.004334163852035999, + "policy/entropy_avg": 0.1855926513671875, + "step": 721, + "val/clipfrac_avg": 0.04114583134651184, + "val/num_eos_tokens": 161, + "val/ratio": 0.989449679851532, + "val/ratio_var": 6.215622852323577e-05 + }, + { + "episode": 11552, + "epoch": 0.5834343434343434, + "eps": 0, + "loss/policy_avg": 0.44041353464126587, + "loss/value_avg": 0.7198789119720459, + "lr": 4.05732104662841e-06, + "objective/entropy": -119.77040100097656, + "objective/kl": 36.42585754394531, + "objective/non_score_reward": -1.8212929964065552, + "objective/rlhf_reward": 13.467769622802734, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008070225012488663, + "policy/clipfrac_avg": 0.007022568956017494, + "policy/entropy_avg": 0.15318524837493896, + "step": 722, + "val/clipfrac_avg": 0.05163312703371048, + "val/num_eos_tokens": 304, + "val/ratio": 0.9899815320968628, + "val/ratio_var": 4.889031333732419e-05 + }, + { + "episode": 11568, + "epoch": 0.5842424242424242, + "eps": 0, + "loss/policy_avg": 0.4866091012954712, + "loss/value_avg": 1.1953067779541016, + "lr": 4.04420685649314e-06, + "objective/entropy": -50.46599578857422, + "objective/kl": 39.113037109375, + "objective/non_score_reward": -1.955651879310608, + "objective/rlhf_reward": 13.911535263061523, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010366745991632342, + "policy/clipfrac_avg": 0.006084335036575794, + "policy/entropy_avg": 0.20149445533752441, + "step": 723, + "val/clipfrac_avg": 0.09489050507545471, + "val/num_eos_tokens": 36, + "val/ratio": 0.9867548942565918, + "val/ratio_var": 6.938407750567421e-05 + }, + { + "episode": 11584, + "epoch": 0.585050505050505, + "eps": 0, + "loss/policy_avg": 0.1589544266462326, + "loss/value_avg": 0.568170964717865, + "lr": 4.031099487368937e-06, + "objective/entropy": -112.03081512451172, + "objective/kl": 37.355255126953125, + "objective/non_score_reward": -1.8677629232406616, + "objective/rlhf_reward": 13.49161148071289, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001035548048093915, + "policy/clipfrac_avg": 0.006865138188004494, + "policy/entropy_avg": 0.20564031600952148, + "step": 724, + "val/clipfrac_avg": 0.05233292281627655, + "val/num_eos_tokens": 347, + "val/ratio": 0.9859846830368042, + "val/ratio_var": 0.00010150864545721561 + }, + { + "episode": 11600, + "epoch": 0.5858585858585859, + "eps": 0, + "loss/policy_avg": -0.19635428488254547, + "loss/value_avg": 0.33023789525032043, + "lr": 4.0179990327964526e-06, + "objective/entropy": -69.79931640625, + "objective/kl": 41.90481948852539, + "objective/non_score_reward": -2.095241069793701, + "objective/rlhf_reward": 14.85007095336914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0010360007872805, + "policy/clipfrac_avg": 0.004560987465083599, + "policy/entropy_avg": 0.20290184020996094, + "step": 725, + "val/clipfrac_avg": 0.08375490456819534, + "val/num_eos_tokens": 261, + "val/ratio": 0.98649001121521, + "val/ratio_var": 6.110723916208372e-05 + }, + { + "episode": 11616, + "epoch": 0.5866666666666667, + "eps": 0, + "loss/policy_avg": 0.0999169647693634, + "loss/value_avg": 1.0625457763671875, + "lr": 4.004905586266988e-06, + "objective/entropy": -104.75285339355469, + "objective/kl": 32.01334762573242, + "objective/non_score_reward": -1.6006674766540527, + "objective/rlhf_reward": 12.766520500183105, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010034965816885233, + "policy/clipfrac_avg": 0.007322699762880802, + "policy/entropy_avg": 0.19988441467285156, + "step": 726, + "val/clipfrac_avg": 0.14562752842903137, + "val/num_eos_tokens": 33, + "val/ratio": 0.9887305498123169, + "val/ratio_var": 7.558859942946583e-05 + }, + { + "episode": 11632, + "epoch": 0.5874747474747475, + "eps": 0, + "loss/policy_avg": 0.004431502893567085, + "loss/value_avg": 0.5651569366455078, + "lr": 3.991819241221836e-06, + "objective/entropy": -53.65510940551758, + "objective/kl": 43.23419952392578, + "objective/non_score_reward": -2.161710262298584, + "objective/rlhf_reward": 12.986727714538574, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001010584644973278, + "policy/clipfrac_avg": 0.006680442951619625, + "policy/entropy_avg": 0.17978286743164062, + "step": 727, + "val/clipfrac_avg": 0.028638603165745735, + "val/num_eos_tokens": 25, + "val/ratio": 0.9880765676498413, + "val/ratio_var": 3.508182999212295e-05 + }, + { + "episode": 11648, + "epoch": 0.5882828282828283, + "eps": 0, + "loss/policy_avg": -0.026804931461811066, + "loss/value_avg": 0.3243325352668762, + "lr": 3.978740091051599e-06, + "objective/entropy": -147.60580444335938, + "objective/kl": 40.000404357910156, + "objective/non_score_reward": -2.0000202655792236, + "objective/rlhf_reward": 13.304666519165039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008220034651458263, + "policy/clipfrac_avg": 0.010559421963989735, + "policy/entropy_avg": 0.15192937850952148, + "step": 728, + "val/clipfrac_avg": 0.047058261930942535, + "val/num_eos_tokens": 93, + "val/ratio": 0.9890267252922058, + "val/ratio_var": 7.159873348427936e-05 + }, + { + "episode": 11664, + "epoch": 0.5890909090909091, + "eps": 0, + "loss/policy_avg": 0.061702877283096313, + "loss/value_avg": 0.25130608677864075, + "lr": 3.965668229095546e-06, + "objective/entropy": -75.26506042480469, + "objective/kl": 34.32489013671875, + "objective/non_score_reward": -1.7162445783615112, + "objective/rlhf_reward": 13.9556303024292, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008753975853323936, + "policy/clipfrac_avg": 0.006253514904528856, + "policy/entropy_avg": 0.17108154296875, + "step": 729, + "val/clipfrac_avg": 0.03564208000898361, + "val/num_eos_tokens": 194, + "val/ratio": 0.9883392453193665, + "val/ratio_var": 5.439678716356866e-05 + }, + { + "episode": 11680, + "epoch": 0.5898989898989899, + "eps": 0, + "loss/policy_avg": 0.5118429064750671, + "loss/value_avg": 0.3809865713119507, + "lr": 3.9526037486409275e-06, + "objective/entropy": -211.5462646484375, + "objective/kl": 38.28752136230469, + "objective/non_score_reward": -1.9143760204315186, + "objective/rlhf_reward": 13.062187194824219, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008200224256142974, + "policy/clipfrac_avg": 0.009272422641515732, + "policy/entropy_avg": 0.15618228912353516, + "step": 730, + "val/clipfrac_avg": 0.13058164715766907, + "val/num_eos_tokens": 191, + "val/ratio": 0.9892928600311279, + "val/ratio_var": 5.910802065045573e-05 + }, + { + "episode": 11696, + "epoch": 0.5907070707070707, + "eps": 0, + "loss/policy_avg": -0.575583279132843, + "loss/value_avg": 0.6588011384010315, + "lr": 3.939546742922318e-06, + "objective/entropy": -155.975341796875, + "objective/kl": 33.536251068115234, + "objective/non_score_reward": -1.6768126487731934, + "objective/rlhf_reward": 14.198186874389648, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006373925716616213, + "policy/clipfrac_avg": 0.011547036468982697, + "policy/entropy_avg": 0.15375041961669922, + "step": 731, + "val/clipfrac_avg": 0.1309666782617569, + "val/num_eos_tokens": 393, + "val/ratio": 0.9912899732589722, + "val/ratio_var": 7.494341116398573e-05 + }, + { + "episode": 11712, + "epoch": 0.5915151515151515, + "eps": 0, + "loss/policy_avg": 0.08975643664598465, + "loss/value_avg": 0.4319133758544922, + "lr": 3.926497305120943e-06, + "objective/entropy": -46.366912841796875, + "objective/kl": 38.771400451660156, + "objective/non_score_reward": -1.9385700225830078, + "objective/rlhf_reward": 14.631742477416992, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010276802349835634, + "policy/clipfrac_avg": 0.004419120494276285, + "policy/entropy_avg": 0.19952774047851562, + "step": 732, + "val/clipfrac_avg": 0.02601846680045128, + "val/num_eos_tokens": 121, + "val/ratio": 0.9856929779052734, + "val/ratio_var": 5.2173218136886135e-05 + }, + { + "episode": 11728, + "epoch": 0.5923232323232324, + "eps": 0, + "loss/policy_avg": -0.5671601295471191, + "loss/value_avg": 1.4009994268417358, + "lr": 3.913455528364027e-06, + "objective/entropy": -30.65240478515625, + "objective/kl": 34.644386291503906, + "objective/non_score_reward": -1.7322194576263428, + "objective/rlhf_reward": 16.127155303955078, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009229708230122924, + "policy/clipfrac_avg": 0.0053136469796299934, + "policy/entropy_avg": 0.1684565544128418, + "step": 733, + "val/clipfrac_avg": 0.16779303550720215, + "val/num_eos_tokens": 257, + "val/ratio": 0.98807692527771, + "val/ratio_var": 4.935808829031885e-05 + }, + { + "episode": 11744, + "epoch": 0.5931313131313132, + "eps": 0, + "loss/policy_avg": 0.1117314025759697, + "loss/value_avg": 0.6027213335037231, + "lr": 3.900421505724119e-06, + "objective/entropy": -115.830078125, + "objective/kl": 38.92997741699219, + "objective/non_score_reward": -1.9464988708496094, + "objective/rlhf_reward": 12.38162612915039, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006995117873884737, + "policy/clipfrac_avg": 0.0067988173104822636, + "policy/entropy_avg": 0.12786293029785156, + "step": 734, + "val/clipfrac_avg": 0.11902365833520889, + "val/num_eos_tokens": 256, + "val/ratio": 0.9913206696510315, + "val/ratio_var": 3.536290387273766e-05 + }, + { + "episode": 11760, + "epoch": 0.593939393939394, + "eps": 0, + "loss/policy_avg": 0.07707646489143372, + "loss/value_avg": 0.8557188510894775, + "lr": 3.887395330218429e-06, + "objective/entropy": -77.05795288085938, + "objective/kl": 34.79220199584961, + "objective/non_score_reward": -1.739610195159912, + "objective/rlhf_reward": 12.510390281677246, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009019911522045732, + "policy/clipfrac_avg": 0.00438590906560421, + "policy/entropy_avg": 0.18432998657226562, + "step": 735, + "val/clipfrac_avg": 0.20910227298736572, + "val/num_eos_tokens": 339, + "val/ratio": 0.9879709482192993, + "val/ratio_var": 7.31311592971906e-05 + }, + { + "episode": 11776, + "epoch": 0.5947474747474748, + "eps": 0, + "loss/policy_avg": 0.005743637681007385, + "loss/value_avg": 0.4823000729084015, + "lr": 3.8743770948081675e-06, + "objective/entropy": -189.58908081054688, + "objective/kl": 40.491024017333984, + "objective/non_score_reward": -2.0245513916015625, + "objective/rlhf_reward": 12.608261108398438, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010324950562790036, + "policy/clipfrac_avg": 0.008821255527436733, + "policy/entropy_avg": 0.1864786148071289, + "step": 736, + "val/clipfrac_avg": 0.03615584969520569, + "val/num_eos_tokens": 146, + "val/ratio": 0.9867680072784424, + "val/ratio_var": 0.00012203787628095597 + }, + { + "episode": 11792, + "epoch": 0.5955555555555555, + "eps": 0, + "loss/policy_avg": 0.11549818515777588, + "loss/value_avg": 0.4191131591796875, + "lr": 3.861366892397878e-06, + "objective/entropy": -53.57340621948242, + "objective/kl": 47.2681999206543, + "objective/non_score_reward": -2.363409996032715, + "objective/rlhf_reward": 11.238153457641602, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0010416920995339751, + "policy/clipfrac_avg": 0.006328020244836807, + "policy/entropy_avg": 0.21573257446289062, + "step": 737, + "val/clipfrac_avg": 0.05454040318727493, + "val/num_eos_tokens": 145, + "val/ratio": 0.9863517880439758, + "val/ratio_var": 5.17384396516718e-05 + }, + { + "episode": 11808, + "epoch": 0.5963636363636363, + "eps": 0, + "loss/policy_avg": 0.09252296388149261, + "loss/value_avg": 0.24368464946746826, + "lr": 3.848364815834782e-06, + "objective/entropy": -9.551445007324219, + "objective/kl": 41.81903839111328, + "objective/non_score_reward": -2.090951919555664, + "objective/rlhf_reward": 15.362173080444336, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.001077502267435193, + "policy/clipfrac_avg": 0.007244379259645939, + "policy/entropy_avg": 0.208465576171875, + "step": 738, + "val/clipfrac_avg": 0.046727634966373444, + "val/num_eos_tokens": 70, + "val/ratio": 0.9859708547592163, + "val/ratio_var": 4.752089807880111e-05 + }, + { + "episode": 11824, + "epoch": 0.5971717171717171, + "eps": 0, + "loss/policy_avg": -0.47144564986228943, + "loss/value_avg": 0.9659825563430786, + "lr": 3.835370957908108e-06, + "objective/entropy": -109.4366226196289, + "objective/kl": 32.577239990234375, + "objective/non_score_reward": -1.6288621425628662, + "objective/rlhf_reward": 13.332075119018555, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009311667527072132, + "policy/clipfrac_avg": 0.007746369112282991, + "policy/entropy_avg": 0.2024822235107422, + "step": 739, + "val/clipfrac_avg": 0.1542586386203766, + "val/num_eos_tokens": 140, + "val/ratio": 0.9888983964920044, + "val/ratio_var": 8.097994577838108e-05 + }, + { + "episode": 11840, + "epoch": 0.597979797979798, + "eps": 0, + "loss/policy_avg": 0.3934873938560486, + "loss/value_avg": 0.5160516500473022, + "lr": 3.822385411348429e-06, + "objective/entropy": -82.36312103271484, + "objective/kl": 38.616180419921875, + "objective/non_score_reward": -1.9308089017868042, + "objective/rlhf_reward": 12.725440979003906, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008991706417873502, + "policy/clipfrac_avg": 0.01843995228409767, + "policy/entropy_avg": 0.2245502471923828, + "step": 740, + "val/clipfrac_avg": 0.0067659346386790276, + "val/num_eos_tokens": 30, + "val/ratio": 0.988348662853241, + "val/ratio_var": 5.281575795379467e-05 + }, + { + "episode": 11856, + "epoch": 0.5987878787878788, + "eps": 0, + "loss/policy_avg": -0.06747608631849289, + "loss/value_avg": 0.3114325702190399, + "lr": 3.809408268827009e-06, + "objective/entropy": -89.22055053710938, + "objective/kl": 46.57951736450195, + "objective/non_score_reward": -2.3289759159088135, + "objective/rlhf_reward": 13.483524322509766, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000996086047962308, + "policy/clipfrac_avg": 0.005896539893001318, + "policy/entropy_avg": 0.19659423828125, + "step": 741, + "val/clipfrac_avg": 0.0699368417263031, + "val/num_eos_tokens": 107, + "val/ratio": 0.9875314235687256, + "val/ratio_var": 4.793483094545081e-05 + }, + { + "episode": 11872, + "epoch": 0.5995959595959596, + "eps": 0, + "loss/policy_avg": -0.2737995684146881, + "loss/value_avg": 0.4772384762763977, + "lr": 3.7964396229551365e-06, + "objective/entropy": -104.62603759765625, + "objective/kl": 36.626800537109375, + "objective/non_score_reward": -1.831339955329895, + "objective/rlhf_reward": 14.778034210205078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008760478813201189, + "policy/clipfrac_avg": 0.005249073728919029, + "policy/entropy_avg": 0.17531394958496094, + "step": 742, + "val/clipfrac_avg": 0.06328597664833069, + "val/num_eos_tokens": 91, + "val/ratio": 0.9885821342468262, + "val/ratio_var": 5.911157131777145e-05 + }, + { + "episode": 11888, + "epoch": 0.6004040404040404, + "eps": 0, + "loss/policy_avg": -0.11605305969715118, + "loss/value_avg": 0.30427810549736023, + "lr": 3.783479566283457e-06, + "objective/entropy": -123.75762176513672, + "objective/kl": 35.787086486816406, + "objective/non_score_reward": -1.7893544435501099, + "objective/rlhf_reward": 14.86689567565918, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007770034717395902, + "policy/clipfrac_avg": 0.0056176697835326195, + "policy/entropy_avg": 0.152313232421875, + "step": 743, + "val/clipfrac_avg": 0.048395052552223206, + "val/num_eos_tokens": 386, + "val/ratio": 0.9905653595924377, + "val/ratio_var": 5.709798642783426e-05 + }, + { + "episode": 11904, + "epoch": 0.6012121212121212, + "eps": 0, + "loss/policy_avg": -0.02294028550386429, + "loss/value_avg": 0.7253668904304504, + "lr": 3.7705281913013286e-06, + "objective/entropy": -39.3416633605957, + "objective/kl": 43.74962615966797, + "objective/non_score_reward": -2.18748140335083, + "objective/rlhf_reward": 12.648456573486328, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010895703453570604, + "policy/clipfrac_avg": 0.0052008070051670074, + "policy/entropy_avg": 0.22774124145507812, + "step": 744, + "val/clipfrac_avg": 0.01829145848751068, + "val/num_eos_tokens": 71, + "val/ratio": 0.985338568687439, + "val/ratio_var": 6.19284255662933e-05 + }, + { + "episode": 11920, + "epoch": 0.602020202020202, + "eps": 0, + "loss/policy_avg": 0.1328388750553131, + "loss/value_avg": 1.3390746116638184, + "lr": 3.7575855904361447e-06, + "objective/entropy": -79.38550567626953, + "objective/kl": 33.780616760253906, + "objective/non_score_reward": -1.6890308856964111, + "objective/rlhf_reward": 14.178156852722168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008661438478156924, + "policy/clipfrac_avg": 0.003558866214007139, + "policy/entropy_avg": 0.15964603424072266, + "step": 745, + "val/clipfrac_avg": 0.05471283197402954, + "val/num_eos_tokens": 81, + "val/ratio": 0.9897934198379517, + "val/ratio_var": 8.215933485189453e-05 + }, + { + "episode": 11936, + "epoch": 0.6028282828282828, + "eps": 0, + "loss/policy_avg": -0.29737263917922974, + "loss/value_avg": 0.6314270496368408, + "lr": 3.7446518560526903e-06, + "objective/entropy": -124.25365447998047, + "objective/kl": 42.908660888671875, + "objective/non_score_reward": -2.145432949066162, + "objective/rlhf_reward": 11.87019157409668, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009470224613323808, + "policy/clipfrac_avg": 0.006878838874399662, + "policy/entropy_avg": 0.1818103790283203, + "step": 746, + "val/clipfrac_avg": 0.0798804834485054, + "val/num_eos_tokens": 329, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 8.261724724434316e-05 + }, + { + "episode": 11952, + "epoch": 0.6036363636363636, + "eps": 0, + "loss/policy_avg": 0.12454540282487869, + "loss/value_avg": 0.4487236738204956, + "lr": 3.731727080452464e-06, + "objective/entropy": -72.31340026855469, + "objective/kl": 43.510276794433594, + "objective/non_score_reward": -2.175513982772827, + "objective/rlhf_reward": 12.347923278808594, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009941391181200743, + "policy/clipfrac_avg": 0.00658874586224556, + "policy/entropy_avg": 0.16569900512695312, + "step": 747, + "val/clipfrac_avg": 0.1540203094482422, + "val/num_eos_tokens": 77, + "val/ratio": 0.9888790845870972, + "val/ratio_var": 2.4041321012191474e-05 + }, + { + "episode": 11968, + "epoch": 0.6044444444444445, + "eps": 0, + "loss/policy_avg": -0.1547861397266388, + "loss/value_avg": 0.49022629857063293, + "lr": 3.7188113558730403e-06, + "objective/entropy": -103.35200500488281, + "objective/kl": 35.28123474121094, + "objective/non_score_reward": -1.764061689376831, + "objective/rlhf_reward": 13.27500057220459, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008191847009584308, + "policy/clipfrac_avg": 0.006343110464513302, + "policy/entropy_avg": 0.14904117584228516, + "step": 748, + "val/clipfrac_avg": 0.04430587589740753, + "val/num_eos_tokens": 124, + "val/ratio": 0.9908789396286011, + "val/ratio_var": 4.545380579656921e-05 + }, + { + "episode": 11984, + "epoch": 0.6052525252525253, + "eps": 0, + "loss/policy_avg": 0.11601711809635162, + "loss/value_avg": 0.6175556182861328, + "lr": 3.705904774487396e-06, + "objective/entropy": -111.66493225097656, + "objective/kl": 36.76349639892578, + "objective/non_score_reward": -1.838174819946289, + "objective/rlhf_reward": 13.693075180053711, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008918631356209517, + "policy/clipfrac_avg": 0.010910662822425365, + "policy/entropy_avg": 0.16454505920410156, + "step": 749, + "val/clipfrac_avg": 0.06755826622247696, + "val/num_eos_tokens": 358, + "val/ratio": 0.9886500835418701, + "val/ratio_var": 6.255816697375849e-05 + }, + { + "episode": 12000, + "epoch": 0.6060606060606061, + "eps": 0, + "loss/policy_avg": -0.04627024754881859, + "loss/value_avg": 0.6224689483642578, + "lr": 3.6930074284032613e-06, + "objective/entropy": -63.2913703918457, + "objective/kl": 32.27269744873047, + "objective/non_score_reward": -1.6136349439620972, + "objective/rlhf_reward": 14.644177436828613, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009490256779827178, + "policy/clipfrac_avg": 0.0068245502188801765, + "policy/entropy_avg": 0.17229938507080078, + "step": 750, + "val/clipfrac_avg": 0.18630267679691315, + "val/num_eos_tokens": 11, + "val/ratio": 0.9878020882606506, + "val/ratio_var": 5.455148493638262e-05 + }, + { + "episode": 12016, + "epoch": 0.6068686868686869, + "eps": 0, + "loss/policy_avg": -0.17890788614749908, + "loss/value_avg": 0.7140947580337524, + "lr": 3.6801194096624515e-06, + "objective/entropy": -54.77946472167969, + "objective/kl": 34.44614028930664, + "objective/non_score_reward": -1.7223069667816162, + "objective/rlhf_reward": 14.894880294799805, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011779924388974905, + "policy/clipfrac_avg": 0.024402525275945663, + "policy/entropy_avg": 0.2098255157470703, + "step": 751, + "val/clipfrac_avg": 0.04182715713977814, + "val/num_eos_tokens": 42, + "val/ratio": 0.9854939579963684, + "val/ratio_var": 0.00014133901277091354 + }, + { + "episode": 12032, + "epoch": 0.6076767676767677, + "eps": 0, + "loss/policy_avg": -0.0015007667243480682, + "loss/value_avg": 0.2544102668762207, + "lr": 3.6672408102402247e-06, + "objective/entropy": -70.776611328125, + "objective/kl": 37.67994689941406, + "objective/non_score_reward": -1.8839974403381348, + "objective/rlhf_reward": 12.108190536499023, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009373503271490335, + "policy/clipfrac_avg": 0.005210112780332565, + "policy/entropy_avg": 0.18710041046142578, + "step": 752, + "val/clipfrac_avg": 0.021128622815012932, + "val/num_eos_tokens": 218, + "val/ratio": 0.9874061942100525, + "val/ratio_var": 5.212551332078874e-05 + }, + { + "episode": 12048, + "epoch": 0.6084848484848485, + "eps": 0, + "loss/policy_avg": 0.05351484939455986, + "loss/value_avg": 0.28849858045578003, + "lr": 3.654371722044616e-06, + "objective/entropy": -89.17417907714844, + "objective/kl": 44.5589714050293, + "objective/non_score_reward": -2.2279486656188965, + "objective/rlhf_reward": 12.178300857543945, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009375615045428276, + "policy/clipfrac_avg": 0.005882620811462402, + "policy/entropy_avg": 0.18878555297851562, + "step": 753, + "val/clipfrac_avg": 0.021566759794950485, + "val/num_eos_tokens": 120, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 5.727734605898149e-05 + }, + { + "episode": 12064, + "epoch": 0.6092929292929293, + "eps": 0, + "loss/policy_avg": -0.12424173951148987, + "loss/value_avg": 0.359388530254364, + "lr": 3.641512236915783e-06, + "objective/entropy": -114.3979263305664, + "objective/kl": 43.28697967529297, + "objective/non_score_reward": -2.16434907913208, + "objective/rlhf_reward": 14.226276397705078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000866133370436728, + "policy/clipfrac_avg": 0.0050315093249082565, + "policy/entropy_avg": 0.16837501525878906, + "step": 754, + "val/clipfrac_avg": 0.1426960825920105, + "val/num_eos_tokens": 391, + "val/ratio": 0.9894628524780273, + "val/ratio_var": 5.016153590986505e-05 + }, + { + "episode": 12080, + "epoch": 0.6101010101010101, + "eps": 0, + "loss/policy_avg": -0.2821425795555115, + "loss/value_avg": 0.38819342851638794, + "lr": 3.6286624466253496e-06, + "objective/entropy": -164.57290649414062, + "objective/kl": 38.83340072631836, + "objective/non_score_reward": -1.9416700601577759, + "objective/rlhf_reward": 14.245829582214355, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008140793070197105, + "policy/clipfrac_avg": 0.00435154652222991, + "policy/entropy_avg": 0.15729975700378418, + "step": 755, + "val/clipfrac_avg": 0.09525477886199951, + "val/num_eos_tokens": 336, + "val/ratio": 0.989446222782135, + "val/ratio_var": 5.3185020078672096e-05 + }, + { + "episode": 12096, + "epoch": 0.610909090909091, + "eps": 0, + "loss/policy_avg": 0.05094696208834648, + "loss/value_avg": 0.4260476529598236, + "lr": 3.6158224428757538e-06, + "objective/entropy": -61.176116943359375, + "objective/kl": 32.80990219116211, + "objective/non_score_reward": -1.6404950618743896, + "objective/rlhf_reward": 16.24231719970703, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009597806492820382, + "policy/clipfrac_avg": 0.0061972904950380325, + "policy/entropy_avg": 0.16566753387451172, + "step": 756, + "val/clipfrac_avg": 0.01815173029899597, + "val/num_eos_tokens": 163, + "val/ratio": 0.9894044399261475, + "val/ratio_var": 4.711053770733997e-05 + }, + { + "episode": 12112, + "epoch": 0.6117171717171718, + "eps": 0, + "loss/policy_avg": -0.040469832718372345, + "loss/value_avg": 0.3746616244316101, + "lr": 3.602992317299594e-06, + "objective/entropy": -52.30070877075195, + "objective/kl": 34.218990325927734, + "objective/non_score_reward": -1.7109496593475342, + "objective/rlhf_reward": 13.273425102233887, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009450222714804113, + "policy/clipfrac_avg": 0.0049097794108092785, + "policy/entropy_avg": 0.1805877685546875, + "step": 757, + "val/clipfrac_avg": 0.09231511503458023, + "val/num_eos_tokens": 105, + "val/ratio": 0.9877369403839111, + "val/ratio_var": 4.407649248605594e-05 + }, + { + "episode": 12128, + "epoch": 0.6125252525252525, + "eps": 0, + "loss/policy_avg": -0.0790981650352478, + "loss/value_avg": 0.289002388715744, + "lr": 3.5901721614589725e-06, + "objective/entropy": -33.08611297607422, + "objective/kl": 42.89472961425781, + "objective/non_score_reward": -2.1447365283966064, + "objective/rlhf_reward": 14.792763710021973, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011628612410277128, + "policy/clipfrac_avg": 0.00601824838668108, + "policy/entropy_avg": 0.21850204467773438, + "step": 758, + "val/clipfrac_avg": 0.05997534096240997, + "val/num_eos_tokens": 208, + "val/ratio": 0.9858998656272888, + "val/ratio_var": 4.6025765186641365e-05 + }, + { + "episode": 12144, + "epoch": 0.6133333333333333, + "eps": 0, + "loss/policy_avg": 0.08223091810941696, + "loss/value_avg": 0.4521709978580475, + "lr": 3.5773620668448384e-06, + "objective/entropy": -57.002418518066406, + "objective/kl": 34.86107635498047, + "objective/non_score_reward": -1.743053674697876, + "objective/rlhf_reward": 12.421009063720703, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009696290362626314, + "policy/clipfrac_avg": 0.005794525146484375, + "policy/entropy_avg": 0.1826324462890625, + "step": 759, + "val/clipfrac_avg": 0.03934604302048683, + "val/num_eos_tokens": 137, + "val/ratio": 0.9880379438400269, + "val/ratio_var": 4.820780668524094e-05 + }, + { + "episode": 12160, + "epoch": 0.6141414141414141, + "eps": 0, + "loss/policy_avg": 0.012903919443488121, + "loss/value_avg": 0.22128555178642273, + "lr": 3.564562124876344e-06, + "objective/entropy": -58.77183532714844, + "objective/kl": 40.15751647949219, + "objective/non_score_reward": -2.007875919342041, + "objective/rlhf_reward": 13.171812057495117, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009755170904099941, + "policy/clipfrac_avg": 0.0075699929147958755, + "policy/entropy_avg": 0.1793661117553711, + "step": 760, + "val/clipfrac_avg": 0.007695037871599197, + "val/num_eos_tokens": 194, + "val/ratio": 0.9879107475280762, + "val/ratio_var": 9.122795017901808e-05 + }, + { + "episode": 12176, + "epoch": 0.6149494949494949, + "eps": 0, + "loss/policy_avg": 0.23269692063331604, + "loss/value_avg": 0.6034168004989624, + "lr": 3.5517724269001853e-06, + "objective/entropy": -92.11294555664062, + "objective/kl": 36.40907287597656, + "objective/non_score_reward": -1.8204538822174072, + "objective/rlhf_reward": 13.468608856201172, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009949133964255452, + "policy/clipfrac_avg": 0.012031201273202896, + "policy/entropy_avg": 0.19497203826904297, + "step": 761, + "val/clipfrac_avg": 0.03142271935939789, + "val/num_eos_tokens": 210, + "val/ratio": 0.9871051907539368, + "val/ratio_var": 7.474326412193477e-05 + }, + { + "episode": 12192, + "epoch": 0.6157575757575757, + "eps": 0, + "loss/policy_avg": -0.014487015083432198, + "loss/value_avg": 1.0809375047683716, + "lr": 3.538993064189954e-06, + "objective/entropy": -42.83490753173828, + "objective/kl": 32.94031524658203, + "objective/non_score_reward": -1.6470158100128174, + "objective/rlhf_reward": 13.567828178405762, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0011591427028179169, + "policy/clipfrac_avg": 0.004484353121370077, + "policy/entropy_avg": 0.21469688415527344, + "step": 762, + "val/clipfrac_avg": 0.1017560362815857, + "val/num_eos_tokens": 453, + "val/ratio": 0.9864501953125, + "val/ratio_var": 8.947903552325442e-05 + }, + { + "episode": 12208, + "epoch": 0.6165656565656565, + "eps": 0, + "loss/policy_avg": -0.0410991832613945, + "loss/value_avg": 1.3014378547668457, + "lr": 3.526224127945479e-06, + "objective/entropy": -71.51974487304688, + "objective/kl": 31.28769302368164, + "objective/non_score_reward": -1.5643846988677979, + "objective/rlhf_reward": 15.216865539550781, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007811100222170353, + "policy/clipfrac_avg": 0.0053521376103162766, + "policy/entropy_avg": 0.16764509677886963, + "step": 763, + "val/clipfrac_avg": 0.059704042971134186, + "val/num_eos_tokens": 203, + "val/ratio": 0.9894566535949707, + "val/ratio_var": 6.841208232799545e-05 + }, + { + "episode": 12224, + "epoch": 0.6173737373737374, + "eps": 0, + "loss/policy_avg": 0.112126424908638, + "loss/value_avg": 0.725554347038269, + "lr": 3.5134657092921877e-06, + "objective/entropy": -99.9854736328125, + "objective/kl": 41.09782028198242, + "objective/non_score_reward": -2.0548908710479736, + "objective/rlhf_reward": 10.624795913696289, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009035035036504269, + "policy/clipfrac_avg": 0.010427422821521759, + "policy/entropy_avg": 0.17998504638671875, + "step": 764, + "val/clipfrac_avg": 0.12764863669872284, + "val/num_eos_tokens": 226, + "val/ratio": 0.988477349281311, + "val/ratio_var": 8.909796451916918e-05 + }, + { + "episode": 12240, + "epoch": 0.6181818181818182, + "eps": 0, + "loss/policy_avg": 0.37758830189704895, + "loss/value_avg": 0.7465800046920776, + "lr": 3.500717899280442e-06, + "objective/entropy": -135.23390197753906, + "objective/kl": 41.466583251953125, + "objective/non_score_reward": -2.073329210281372, + "objective/rlhf_reward": 12.94229507446289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009621024364605546, + "policy/clipfrac_avg": 0.00628144945949316, + "policy/entropy_avg": 0.17942047119140625, + "step": 765, + "val/clipfrac_avg": 0.10096652805805206, + "val/num_eos_tokens": 233, + "val/ratio": 0.9877568483352661, + "val/ratio_var": 7.067532715154812e-05 + }, + { + "episode": 12256, + "epoch": 0.618989898989899, + "eps": 0, + "loss/policy_avg": -0.10782650113105774, + "loss/value_avg": 0.4996359944343567, + "lr": 3.4879807888849026e-06, + "objective/entropy": -86.19502258300781, + "objective/kl": 39.61420440673828, + "objective/non_score_reward": -1.9807100296020508, + "objective/rlhf_reward": 13.706789016723633, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009516358841210604, + "policy/clipfrac_avg": 0.01008635200560093, + "policy/entropy_avg": 0.17633438110351562, + "step": 766, + "val/clipfrac_avg": 0.05575938522815704, + "val/num_eos_tokens": 426, + "val/ratio": 0.9884860515594482, + "val/ratio_var": 5.413329563452862e-05 + }, + { + "episode": 12272, + "epoch": 0.6197979797979798, + "eps": 0, + "loss/policy_avg": 0.0611935630440712, + "loss/value_avg": 0.40072089433670044, + "lr": 3.475254469003865e-06, + "objective/entropy": -78.26728057861328, + "objective/kl": 38.48719787597656, + "objective/non_score_reward": -1.9243600368499756, + "objective/rlhf_reward": 11.661577224731445, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009466722258366644, + "policy/clipfrac_avg": 0.0042708818800747395, + "policy/entropy_avg": 0.17498779296875, + "step": 767, + "val/clipfrac_avg": 0.1151452288031578, + "val/num_eos_tokens": 415, + "val/ratio": 0.9874659776687622, + "val/ratio_var": 7.443072536261752e-05 + }, + { + "episode": 12288, + "epoch": 0.6206060606060606, + "eps": 0, + "loss/policy_avg": -0.07818017899990082, + "loss/value_avg": 0.588293731212616, + "lr": 3.4625390304586224e-06, + "objective/entropy": -45.31693649291992, + "objective/kl": 41.06260681152344, + "objective/non_score_reward": -2.0531303882598877, + "objective/rlhf_reward": 13.860933303833008, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0011651598615571856, + "policy/clipfrac_avg": 0.0047182347625494, + "policy/entropy_avg": 0.2090606689453125, + "step": 768, + "val/clipfrac_avg": 0.16338574886322021, + "val/num_eos_tokens": 9, + "val/ratio": 0.986497700214386, + "val/ratio_var": 8.175932453013957e-05 + }, + { + "episode": 12304, + "epoch": 0.6214141414141414, + "eps": 0, + "loss/policy_avg": -0.3265213966369629, + "loss/value_avg": 0.5878744125366211, + "lr": 3.4498345639928167e-06, + "objective/entropy": -103.96510314941406, + "objective/kl": 35.794551849365234, + "objective/non_score_reward": -1.7897276878356934, + "objective/rlhf_reward": 13.507146835327148, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010084104724228382, + "policy/clipfrac_avg": 0.0070042358711361885, + "policy/entropy_avg": 0.19075870513916016, + "step": 769, + "val/clipfrac_avg": 0.12878292798995972, + "val/num_eos_tokens": 8, + "val/ratio": 0.9861502051353455, + "val/ratio_var": 8.973718649940565e-05 + }, + { + "episode": 12320, + "epoch": 0.6222222222222222, + "eps": 0, + "loss/policy_avg": -0.24864813685417175, + "loss/value_avg": 0.703667402267456, + "lr": 3.4371411602717785e-06, + "objective/entropy": -92.59934997558594, + "objective/kl": 31.87890625, + "objective/non_score_reward": -1.5939452648162842, + "objective/rlhf_reward": 14.429492950439453, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007836055592633784, + "policy/clipfrac_avg": 0.004061062820255756, + "policy/entropy_avg": 0.13613224029541016, + "step": 770, + "val/clipfrac_avg": 0.14356926083564758, + "val/num_eos_tokens": 22, + "val/ratio": 0.9909623265266418, + "val/ratio_var": 6.252773891901597e-05 + }, + { + "episode": 12336, + "epoch": 0.623030303030303, + "eps": 0, + "loss/policy_avg": -0.11106158792972565, + "loss/value_avg": 0.7719787359237671, + "lr": 3.424458909881897e-06, + "objective/entropy": -201.825927734375, + "objective/kl": 32.05082321166992, + "objective/non_score_reward": -1.602541208267212, + "objective/rlhf_reward": 13.842771530151367, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009224842651747167, + "policy/clipfrac_avg": 0.004090993665158749, + "policy/entropy_avg": 0.19452500343322754, + "step": 771, + "val/clipfrac_avg": 0.06381858140230179, + "val/num_eos_tokens": 137, + "val/ratio": 0.9873221516609192, + "val/ratio_var": 0.00010800129530252889 + }, + { + "episode": 12352, + "epoch": 0.6238383838383839, + "eps": 0, + "loss/policy_avg": -0.3147702217102051, + "loss/value_avg": 1.0198739767074585, + "lr": 3.411787903329963e-06, + "objective/entropy": -97.80130767822266, + "objective/kl": 28.94087791442871, + "objective/non_score_reward": -1.4470438957214355, + "objective/rlhf_reward": 13.572486877441406, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008233180269598961, + "policy/clipfrac_avg": 0.007523335050791502, + "policy/entropy_avg": 0.17452716827392578, + "step": 772, + "val/clipfrac_avg": 0.021688329055905342, + "val/num_eos_tokens": 116, + "val/ratio": 0.9885031580924988, + "val/ratio_var": 6.793635111534968e-05 + }, + { + "episode": 12368, + "epoch": 0.6246464646464647, + "eps": 0, + "loss/policy_avg": -0.019919998943805695, + "loss/value_avg": 0.31691867113113403, + "lr": 3.3991282310425276e-06, + "objective/entropy": -61.794166564941406, + "objective/kl": 33.001129150390625, + "objective/non_score_reward": -1.6500564813613892, + "objective/rlhf_reward": 12.943693161010742, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009417567634955049, + "policy/clipfrac_avg": 0.005282199941575527, + "policy/entropy_avg": 0.1732311248779297, + "step": 773, + "val/clipfrac_avg": 0.05137361213564873, + "val/num_eos_tokens": 13, + "val/ratio": 0.9889352321624756, + "val/ratio_var": 3.9570189983351156e-05 + }, + { + "episode": 12384, + "epoch": 0.6254545454545455, + "eps": 0, + "loss/policy_avg": 0.010051839053630829, + "loss/value_avg": 0.5414019227027893, + "lr": 3.386479983365251e-06, + "objective/entropy": -73.91504669189453, + "objective/kl": 29.43255615234375, + "objective/non_score_reward": -1.4716278314590454, + "objective/rlhf_reward": 13.293996810913086, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0010180522222071886, + "policy/clipfrac_avg": 0.0050889430567622185, + "policy/entropy_avg": 0.2153911590576172, + "step": 774, + "val/clipfrac_avg": 0.041913747787475586, + "val/num_eos_tokens": 214, + "val/ratio": 0.986315906047821, + "val/ratio_var": 8.102346328087151e-05 + }, + { + "episode": 12400, + "epoch": 0.6262626262626263, + "eps": 0, + "loss/policy_avg": -0.040474407374858856, + "loss/value_avg": 0.30427631735801697, + "lr": 3.3738432505622653e-06, + "objective/entropy": -75.78406524658203, + "objective/kl": 31.846160888671875, + "objective/non_score_reward": -1.5923081636428833, + "objective/rlhf_reward": 11.946754455566406, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009044273756444454, + "policy/clipfrac_avg": 0.005002772435545921, + "policy/entropy_avg": 0.18775272369384766, + "step": 775, + "val/clipfrac_avg": 0.04153668135404587, + "val/num_eos_tokens": 243, + "val/ratio": 0.9877529144287109, + "val/ratio_var": 5.462392800836824e-05 + }, + { + "episode": 12416, + "epoch": 0.6270707070707071, + "eps": 0, + "loss/policy_avg": -0.25257226824760437, + "loss/value_avg": 0.47636741399765015, + "lr": 3.3612181228155237e-06, + "objective/entropy": -97.84140014648438, + "objective/kl": 35.974266052246094, + "objective/non_score_reward": -1.798713207244873, + "objective/rlhf_reward": 13.685661315917969, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009363207500427961, + "policy/clipfrac_avg": 0.0035052683670073748, + "policy/entropy_avg": 0.18935298919677734, + "step": 776, + "val/clipfrac_avg": 0.07260937243700027, + "val/num_eos_tokens": 26, + "val/ratio": 0.9877586364746094, + "val/ratio_var": 0.00010292992374161258 + }, + { + "episode": 12432, + "epoch": 0.6278787878787879, + "eps": 0, + "loss/policy_avg": 0.41764986515045166, + "loss/value_avg": 0.8629061579704285, + "lr": 3.3486046902241663e-06, + "objective/entropy": -40.24072265625, + "objective/kl": 29.197486877441406, + "objective/non_score_reward": -1.4598743915557861, + "objective/rlhf_reward": 12.883874893188477, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010831760009750724, + "policy/clipfrac_avg": 0.02700810506939888, + "policy/entropy_avg": 0.16212844848632812, + "step": 777, + "val/clipfrac_avg": 0.08423963189125061, + "val/num_eos_tokens": 192, + "val/ratio": 0.9885735511779785, + "val/ratio_var": 8.724266081117094e-05 + }, + { + "episode": 12448, + "epoch": 0.6286868686868687, + "eps": 0, + "loss/policy_avg": 0.2242026925086975, + "loss/value_avg": 0.9421164989471436, + "lr": 3.336003042803861e-06, + "objective/entropy": -69.478271484375, + "objective/kl": 33.57006072998047, + "objective/non_score_reward": -1.6785030364990234, + "objective/rlhf_reward": 14.055871963500977, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008899248205125332, + "policy/clipfrac_avg": 0.006825867109000683, + "policy/entropy_avg": 0.14767837524414062, + "step": 778, + "val/clipfrac_avg": 0.1501787304878235, + "val/num_eos_tokens": 109, + "val/ratio": 0.9905418753623962, + "val/ratio_var": 4.142183388466947e-05 + }, + { + "episode": 12464, + "epoch": 0.6294949494949494, + "eps": 0, + "loss/policy_avg": 0.03614666312932968, + "loss/value_avg": 0.7387021780014038, + "lr": 3.3234132704861786e-06, + "objective/entropy": -182.4083251953125, + "objective/kl": 31.944690704345703, + "objective/non_score_reward": -1.5972344875335693, + "objective/rlhf_reward": 12.043390274047852, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.000871892087161541, + "policy/clipfrac_avg": 0.004613575059920549, + "policy/entropy_avg": 0.1733226776123047, + "step": 779, + "val/clipfrac_avg": 0.1070820540189743, + "val/num_eos_tokens": 28, + "val/ratio": 0.9880519509315491, + "val/ratio_var": 9.137512097368017e-05 + }, + { + "episode": 12480, + "epoch": 0.6303030303030303, + "eps": 0, + "loss/policy_avg": 0.491243839263916, + "loss/value_avg": 1.7449349164962769, + "lr": 3.310835463117942e-06, + "objective/entropy": -80.43102264404297, + "objective/kl": 22.97904396057129, + "objective/non_score_reward": -1.1489522457122803, + "objective/rlhf_reward": 13.28073501586914, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000753985601477325, + "policy/clipfrac_avg": 0.013304230757057667, + "policy/entropy_avg": 0.16235482692718506, + "step": 780, + "val/clipfrac_avg": 0.12828537821769714, + "val/num_eos_tokens": 261, + "val/ratio": 0.991666316986084, + "val/ratio_var": 4.57862151961308e-05 + }, + { + "episode": 12496, + "epoch": 0.6311111111111111, + "eps": 0, + "loss/policy_avg": -0.02106083557009697, + "loss/value_avg": 0.26465463638305664, + "lr": 3.2982697104605855e-06, + "objective/entropy": -96.58609008789062, + "objective/kl": 37.96983337402344, + "objective/non_score_reward": -1.898491621017456, + "objective/rlhf_reward": 12.742133140563965, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008919077808968723, + "policy/clipfrac_avg": 0.004138793330639601, + "policy/entropy_avg": 0.176055908203125, + "step": 781, + "val/clipfrac_avg": 0.022776879370212555, + "val/num_eos_tokens": 232, + "val/ratio": 0.9880313873291016, + "val/ratio_var": 5.30094766872935e-05 + }, + { + "episode": 12512, + "epoch": 0.6319191919191919, + "eps": 0, + "loss/policy_avg": -0.21595685184001923, + "loss/value_avg": 0.1668311059474945, + "lr": 3.2857161021895125e-06, + "objective/entropy": -113.97618103027344, + "objective/kl": 30.898021697998047, + "objective/non_score_reward": -1.5449011325836182, + "objective/rlhf_reward": 15.892599105834961, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0008122409926727414, + "policy/clipfrac_avg": 0.007180417887866497, + "policy/entropy_avg": 0.15228557586669922, + "step": 782, + "val/clipfrac_avg": 0.04710119590163231, + "val/num_eos_tokens": 88, + "val/ratio": 0.9905120134353638, + "val/ratio_var": 4.6862616727594286e-05 + }, + { + "episode": 12528, + "epoch": 0.6327272727272727, + "eps": 0, + "loss/policy_avg": -0.02830284647643566, + "loss/value_avg": 0.470698744058609, + "lr": 3.273174727893463e-06, + "objective/entropy": -119.84375762939453, + "objective/kl": 35.46149444580078, + "objective/non_score_reward": -1.773074746131897, + "objective/rlhf_reward": 13.00036334991455, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000846280949190259, + "policy/clipfrac_avg": 0.006307992152869701, + "policy/entropy_avg": 0.15985870361328125, + "step": 783, + "val/clipfrac_avg": 0.09303553402423859, + "val/num_eos_tokens": 242, + "val/ratio": 0.9886760711669922, + "val/ratio_var": 8.341170905623585e-05 + }, + { + "episode": 12544, + "epoch": 0.6335353535353535, + "eps": 0, + "loss/policy_avg": -0.03483296558260918, + "loss/value_avg": 0.2278536707162857, + "lr": 3.2606456770738636e-06, + "objective/entropy": -32.83402633666992, + "objective/kl": 35.053001403808594, + "objective/non_score_reward": -1.7526500225067139, + "objective/rlhf_reward": 13.708287239074707, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010523913661018014, + "policy/clipfrac_avg": 0.005916148889809847, + "policy/entropy_avg": 0.2171039581298828, + "step": 784, + "val/clipfrac_avg": 0.007893375121057034, + "val/num_eos_tokens": 463, + "val/ratio": 0.9859672784805298, + "val/ratio_var": 5.139111090102233e-05 + }, + { + "episode": 12560, + "epoch": 0.6343434343434343, + "eps": 0, + "loss/policy_avg": 0.17171788215637207, + "loss/value_avg": 0.5937256217002869, + "lr": 3.2481290391442034e-06, + "objective/entropy": -109.50154113769531, + "objective/kl": 41.38114929199219, + "objective/non_score_reward": -2.0690574645996094, + "objective/rlhf_reward": 13.82156753540039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009320294484496117, + "policy/clipfrac_avg": 0.005337849725037813, + "policy/entropy_avg": 0.1828467845916748, + "step": 785, + "val/clipfrac_avg": 0.0434221550822258, + "val/num_eos_tokens": 252, + "val/ratio": 0.9880971908569336, + "val/ratio_var": 7.803420157870278e-05 + }, + { + "episode": 12576, + "epoch": 0.6351515151515151, + "eps": 0, + "loss/policy_avg": -0.12326152622699738, + "loss/value_avg": 0.22453081607818604, + "lr": 3.235624903429374e-06, + "objective/entropy": -80.62982940673828, + "objective/kl": 39.33965301513672, + "objective/non_score_reward": -1.9669828414916992, + "objective/rlhf_reward": 14.3767671585083, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009726607240736485, + "policy/clipfrac_avg": 0.005861825309693813, + "policy/entropy_avg": 0.18674850463867188, + "step": 786, + "val/clipfrac_avg": 0.04197417199611664, + "val/num_eos_tokens": 55, + "val/ratio": 0.9867269396781921, + "val/ratio_var": 3.261497477069497e-05 + }, + { + "episode": 12592, + "epoch": 0.635959595959596, + "eps": 0, + "loss/policy_avg": 0.02891068160533905, + "loss/value_avg": 0.2414318025112152, + "lr": 3.2231333591650567e-06, + "objective/entropy": -64.31414031982422, + "objective/kl": 46.58558654785156, + "objective/non_score_reward": -2.3292794227600098, + "objective/rlhf_reward": 14.576970100402832, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009546708315610886, + "policy/clipfrac_avg": 0.00552853848785162, + "policy/entropy_avg": 0.189300537109375, + "step": 787, + "val/clipfrac_avg": 0.0013757861452177167, + "val/num_eos_tokens": 363, + "val/ratio": 0.987205982208252, + "val/ratio_var": 3.5099023079965264e-05 + }, + { + "episode": 12608, + "epoch": 0.6367676767676768, + "eps": 0, + "loss/policy_avg": -0.19538097083568573, + "loss/value_avg": 0.7160595655441284, + "lr": 3.210654495497068e-06, + "objective/entropy": -57.81846237182617, + "objective/kl": 42.85956573486328, + "objective/non_score_reward": -2.1429784297943115, + "objective/rlhf_reward": 12.14999008178711, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0011135776294395328, + "policy/clipfrac_avg": 0.009378955699503422, + "policy/entropy_avg": 0.2034454345703125, + "step": 788, + "val/clipfrac_avg": 0.1690627932548523, + "val/num_eos_tokens": 481, + "val/ratio": 0.9861621856689453, + "val/ratio_var": 9.24188134376891e-05 + }, + { + "episode": 12624, + "epoch": 0.6375757575757576, + "eps": 0, + "loss/policy_avg": 0.048018310219049454, + "loss/value_avg": 0.4312187135219574, + "lr": 3.198188401480734e-06, + "objective/entropy": -76.67124938964844, + "objective/kl": 36.849369049072266, + "objective/non_score_reward": -1.8424686193466187, + "objective/rlhf_reward": 12.25128173828125, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009952555410563946, + "policy/clipfrac_avg": 0.004150367341935635, + "policy/entropy_avg": 0.18071389198303223, + "step": 789, + "val/clipfrac_avg": 0.08804323524236679, + "val/num_eos_tokens": 357, + "val/ratio": 0.9867403507232666, + "val/ratio_var": 0.00013288675108924508 + }, + { + "episode": 12640, + "epoch": 0.6383838383838384, + "eps": 0, + "loss/policy_avg": 0.18971526622772217, + "loss/value_avg": 0.6626036167144775, + "lr": 3.185735166080244e-06, + "objective/entropy": -126.28926086425781, + "objective/kl": 33.96638488769531, + "objective/non_score_reward": -1.6983189582824707, + "objective/rlhf_reward": 12.973556518554688, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0007302669691853225, + "policy/clipfrac_avg": 0.008397979661822319, + "policy/entropy_avg": 0.14786386489868164, + "step": 790, + "val/clipfrac_avg": 0.06916166841983795, + "val/num_eos_tokens": 600, + "val/ratio": 0.9910507798194885, + "val/ratio_var": 5.283964492264204e-05 + }, + { + "episode": 12656, + "epoch": 0.6391919191919192, + "eps": 0, + "loss/policy_avg": -0.4279332160949707, + "loss/value_avg": 0.5117957592010498, + "lr": 3.173294878168025e-06, + "objective/entropy": -75.64164733886719, + "objective/kl": 40.16947555541992, + "objective/non_score_reward": -2.0084738731384277, + "objective/rlhf_reward": 12.86652660369873, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008897786610759795, + "policy/clipfrac_avg": 0.004540015943348408, + "policy/entropy_avg": 0.18616324663162231, + "step": 791, + "val/clipfrac_avg": 0.0338541679084301, + "val/num_eos_tokens": 224, + "val/ratio": 0.9873502254486084, + "val/ratio_var": 6.438903801608831e-05 + }, + { + "episode": 12672, + "epoch": 0.64, + "eps": 0, + "loss/policy_avg": -0.30223172903060913, + "loss/value_avg": 0.7414761185646057, + "lr": 3.160867626524107e-06, + "objective/entropy": -36.1086540222168, + "objective/kl": 35.08636474609375, + "objective/non_score_reward": -1.7543182373046875, + "objective/rlhf_reward": 14.995681762695312, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010164354462176561, + "policy/clipfrac_avg": 0.0033426620066165924, + "policy/entropy_avg": 0.20384597778320312, + "step": 792, + "val/clipfrac_avg": 0.08375494927167892, + "val/num_eos_tokens": 121, + "val/ratio": 0.9862762093544006, + "val/ratio_var": 8.216432615881786e-05 + }, + { + "episode": 12688, + "epoch": 0.6408080808080808, + "eps": 0, + "loss/policy_avg": 0.11556120216846466, + "loss/value_avg": 0.44696569442749023, + "lr": 3.1484534998354832e-06, + "objective/entropy": -52.89218521118164, + "objective/kl": 33.01978302001953, + "objective/non_score_reward": -1.650989294052124, + "objective/rlhf_reward": 12.270885467529297, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009637182811275125, + "policy/clipfrac_avg": 0.006595691200345755, + "policy/entropy_avg": 0.18551349639892578, + "step": 793, + "val/clipfrac_avg": 0.0313958078622818, + "val/num_eos_tokens": 483, + "val/ratio": 0.987555742263794, + "val/ratio_var": 4.9779242544900626e-05 + }, + { + "episode": 12704, + "epoch": 0.6416161616161616, + "eps": 0, + "loss/policy_avg": -0.11985203623771667, + "loss/value_avg": 0.5595318078994751, + "lr": 3.136052586695481e-06, + "objective/entropy": -44.2855110168457, + "objective/kl": 38.74113845825195, + "objective/non_score_reward": -1.9370570182800293, + "objective/rlhf_reward": 13.531692504882812, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009892593370750546, + "policy/clipfrac_avg": 0.006002710200846195, + "policy/entropy_avg": 0.20067405700683594, + "step": 794, + "val/clipfrac_avg": 0.051367055624723434, + "val/num_eos_tokens": 146, + "val/ratio": 0.98707115650177, + "val/ratio_var": 6.210069841472432e-05 + }, + { + "episode": 12720, + "epoch": 0.6424242424242425, + "eps": 0, + "loss/policy_avg": 0.06161116436123848, + "loss/value_avg": 0.20387178659439087, + "lr": 3.12366497560313e-06, + "objective/entropy": -47.063812255859375, + "objective/kl": 36.60810852050781, + "objective/non_score_reward": -1.8304054737091064, + "objective/rlhf_reward": 13.661782264709473, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000998336123302579, + "policy/clipfrac_avg": 0.004183639772236347, + "policy/entropy_avg": 0.19140625, + "step": 795, + "val/clipfrac_avg": 0.08834637701511383, + "val/num_eos_tokens": 342, + "val/ratio": 0.9874628782272339, + "val/ratio_var": 3.3792719477787614e-05 + }, + { + "episode": 12736, + "epoch": 0.6432323232323233, + "eps": 0, + "loss/policy_avg": 0.02652081847190857, + "loss/value_avg": 0.4432865381240845, + "lr": 3.1112907549625304e-06, + "objective/entropy": -104.79203796386719, + "objective/kl": 35.12114715576172, + "objective/non_score_reward": -1.7560575008392334, + "objective/rlhf_reward": 13.486129760742188, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008146626059897244, + "policy/clipfrac_avg": 0.0056906514801084995, + "policy/entropy_avg": 0.1502237319946289, + "step": 796, + "val/clipfrac_avg": 0.061473242938518524, + "val/num_eos_tokens": 748, + "val/ratio": 0.9898946285247803, + "val/ratio_var": 4.619114042725414e-05 + }, + { + "episode": 12752, + "epoch": 0.6440404040404041, + "eps": 0, + "loss/policy_avg": 0.06834741681814194, + "loss/value_avg": 0.5731651782989502, + "lr": 3.098930013082223e-06, + "objective/entropy": -88.31727600097656, + "objective/kl": 38.224117279052734, + "objective/non_score_reward": -1.9112060070037842, + "objective/rlhf_reward": 14.698169708251953, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008200570591725409, + "policy/clipfrac_avg": 0.007693252060562372, + "policy/entropy_avg": 0.1607208251953125, + "step": 797, + "val/clipfrac_avg": 0.11237858235836029, + "val/num_eos_tokens": 240, + "val/ratio": 0.9905504584312439, + "val/ratio_var": 4.706854815594852e-05 + }, + { + "episode": 12768, + "epoch": 0.6448484848484849, + "eps": 0, + "loss/policy_avg": -0.09761235117912292, + "loss/value_avg": 0.7088909149169922, + "lr": 3.0865828381745515e-06, + "objective/entropy": -124.7913589477539, + "objective/kl": 29.702392578125, + "objective/non_score_reward": -1.4851195812225342, + "objective/rlhf_reward": 13.68675422668457, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008782441727817059, + "policy/clipfrac_avg": 0.00598434591665864, + "policy/entropy_avg": 0.16924285888671875, + "step": 798, + "val/clipfrac_avg": 0.030641667544841766, + "val/num_eos_tokens": 454, + "val/ratio": 0.9882635474205017, + "val/ratio_var": 7.515518518630415e-05 + }, + { + "episode": 12784, + "epoch": 0.6456565656565657, + "eps": 0, + "loss/policy_avg": 0.14146611094474792, + "loss/value_avg": 0.7321012616157532, + "lr": 3.074249318355046e-06, + "objective/entropy": -101.5077133178711, + "objective/kl": 25.74894142150879, + "objective/non_score_reward": -1.2874470949172974, + "objective/rlhf_reward": 13.735990524291992, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007512210868299007, + "policy/clipfrac_avg": 0.011982694268226624, + "policy/entropy_avg": 0.15892601013183594, + "step": 799, + "val/clipfrac_avg": 0.012677992694079876, + "val/num_eos_tokens": 12, + "val/ratio": 0.9910134077072144, + "val/ratio_var": 6.09022390563041e-05 + }, + { + "episode": 12800, + "epoch": 0.6464646464646465, + "eps": 0, + "loss/policy_avg": -0.042851828038692474, + "loss/value_avg": 0.3452759385108948, + "lr": 3.0619295416417856e-06, + "objective/entropy": -68.88214874267578, + "objective/kl": 34.023399353027344, + "objective/non_score_reward": -1.7011699676513672, + "objective/rlhf_reward": 12.494142532348633, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010239251423627138, + "policy/clipfrac_avg": 0.010894561186432838, + "policy/entropy_avg": 0.17738628387451172, + "step": 800, + "val/clipfrac_avg": 0.026722928509116173, + "val/num_eos_tokens": 67, + "val/ratio": 0.9888595342636108, + "val/ratio_var": 6.888628558954224e-05 + }, + { + "episode": 12816, + "epoch": 0.6472727272727272, + "eps": 0, + "loss/policy_avg": -0.05754968896508217, + "loss/value_avg": 0.402332067489624, + "lr": 3.049623595954766e-06, + "objective/entropy": -89.62500762939453, + "objective/kl": 32.48891067504883, + "objective/non_score_reward": -1.6244455575942993, + "objective/rlhf_reward": 14.227116584777832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009385538287460804, + "policy/clipfrac_avg": 0.004920909181237221, + "policy/entropy_avg": 0.17440450191497803, + "step": 801, + "val/clipfrac_avg": 0.03176930546760559, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880223274230957, + "val/ratio_var": 5.388163845054805e-05 + }, + { + "episode": 12832, + "epoch": 0.648080808080808, + "eps": 0, + "loss/policy_avg": 0.2483101636171341, + "loss/value_avg": 0.23584425449371338, + "lr": 3.0373315691152856e-06, + "objective/entropy": -89.31193542480469, + "objective/kl": 40.699127197265625, + "objective/non_score_reward": -2.034956455230713, + "objective/rlhf_reward": 12.160355567932129, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0008679171442054212, + "policy/clipfrac_avg": 0.004822667222470045, + "policy/entropy_avg": 0.1660480499267578, + "step": 802, + "val/clipfrac_avg": 0.05813268944621086, + "val/num_eos_tokens": 149, + "val/ratio": 0.9890165328979492, + "val/ratio_var": 5.875104761798866e-05 + }, + { + "episode": 12848, + "epoch": 0.6488888888888888, + "eps": 0, + "loss/policy_avg": -0.10593897104263306, + "loss/value_avg": 0.5456434488296509, + "lr": 3.0250535488453077e-06, + "objective/entropy": -148.4324188232422, + "objective/kl": 38.51166915893555, + "objective/non_score_reward": -1.9255836009979248, + "objective/rlhf_reward": 11.918167114257812, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0007962615345604718, + "policy/clipfrac_avg": 0.0040261996909976006, + "policy/entropy_avg": 0.16094493865966797, + "step": 803, + "val/clipfrac_avg": 0.04293014109134674, + "val/num_eos_tokens": 331, + "val/ratio": 0.9893560409545898, + "val/ratio_var": 6.857667176518589e-05 + }, + { + "episode": 12864, + "epoch": 0.6496969696969697, + "eps": 0, + "loss/policy_avg": 0.06602153182029724, + "loss/value_avg": 0.2823004722595215, + "lr": 3.0127896227668367e-06, + "objective/entropy": -115.89801788330078, + "objective/kl": 40.921504974365234, + "objective/non_score_reward": -2.0460751056671143, + "objective/rlhf_reward": 12.867987632751465, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008664772612974048, + "policy/clipfrac_avg": 0.00966357346624136, + "policy/entropy_avg": 0.151214599609375, + "step": 804, + "val/clipfrac_avg": 0.03888742998242378, + "val/num_eos_tokens": 224, + "val/ratio": 0.9900601506233215, + "val/ratio_var": 5.00909227412194e-05 + }, + { + "episode": 12880, + "epoch": 0.6505050505050505, + "eps": 0, + "loss/policy_avg": -0.39892637729644775, + "loss/value_avg": 0.4240272641181946, + "lr": 3.000539878401296e-06, + "objective/entropy": -77.75199890136719, + "objective/kl": 35.056907653808594, + "objective/non_score_reward": -1.7528454065322876, + "objective/rlhf_reward": 14.00496768951416, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.001023960066959262, + "policy/clipfrac_avg": 0.004629913717508316, + "policy/entropy_avg": 0.20476627349853516, + "step": 805, + "val/clipfrac_avg": 0.07910472899675369, + "val/num_eos_tokens": 219, + "val/ratio": 0.9861233830451965, + "val/ratio_var": 8.373144373763353e-05 + }, + { + "episode": 12896, + "epoch": 0.6513131313131313, + "eps": 0, + "loss/policy_avg": -0.23323668539524078, + "loss/value_avg": 0.4607469439506531, + "lr": 2.9883044031688995e-06, + "objective/entropy": -85.76292419433594, + "objective/kl": 40.88092803955078, + "objective/non_score_reward": -2.044046401977539, + "objective/rlhf_reward": 14.174703598022461, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010081828804686666, + "policy/clipfrac_avg": 0.0041284579783678055, + "policy/entropy_avg": 0.1921253204345703, + "step": 806, + "val/clipfrac_avg": 0.16490784287452698, + "val/num_eos_tokens": 249, + "val/ratio": 0.9875838756561279, + "val/ratio_var": 6.450649379985407e-05 + }, + { + "episode": 12912, + "epoch": 0.6521212121212121, + "eps": 0, + "loss/policy_avg": -0.11845395714044571, + "loss/value_avg": 0.4592535197734833, + "lr": 2.976083284388031e-06, + "objective/entropy": -62.67930221557617, + "objective/kl": 33.47719955444336, + "objective/non_score_reward": -1.6738600730895996, + "objective/rlhf_reward": 12.771451950073242, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011091704946011305, + "policy/clipfrac_avg": 0.00747725460678339, + "policy/entropy_avg": 0.1983814239501953, + "step": 807, + "val/clipfrac_avg": 0.033114466816186905, + "val/num_eos_tokens": 17, + "val/ratio": 0.9877710938453674, + "val/ratio_var": 6.532426050398499e-05 + }, + { + "episode": 12928, + "epoch": 0.6529292929292929, + "eps": 0, + "loss/policy_avg": 0.0469900444149971, + "loss/value_avg": 0.26954370737075806, + "lr": 2.9638766092746207e-06, + "objective/entropy": -88.9464111328125, + "objective/kl": 34.25027847290039, + "objective/non_score_reward": -1.7125139236450195, + "objective/rlhf_reward": 12.56873607635498, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009448326891288161, + "policy/clipfrac_avg": 0.006387755274772644, + "policy/entropy_avg": 0.18566322326660156, + "step": 808, + "val/clipfrac_avg": 0.018177732825279236, + "val/num_eos_tokens": 761, + "val/ratio": 0.9882063269615173, + "val/ratio_var": 5.754105222877115e-05 + }, + { + "episode": 12944, + "epoch": 0.6537373737373737, + "eps": 0, + "loss/policy_avg": 0.00823296420276165, + "loss/value_avg": 0.44794195890426636, + "lr": 2.9516844649415165e-06, + "objective/entropy": -93.69944763183594, + "objective/kl": 40.86418151855469, + "objective/non_score_reward": -2.0432090759277344, + "objective/rlhf_reward": 13.542728424072266, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000799038156401366, + "policy/clipfrac_avg": 0.006408605724573135, + "policy/entropy_avg": 0.14969253540039062, + "step": 809, + "val/clipfrac_avg": 0.08976449072360992, + "val/num_eos_tokens": 215, + "val/ratio": 0.989780068397522, + "val/ratio_var": 6.0727201343979686e-05 + }, + { + "episode": 12960, + "epoch": 0.6545454545454545, + "eps": 0, + "loss/policy_avg": 0.024010583758354187, + "loss/value_avg": 0.3378852605819702, + "lr": 2.9395069383978725e-06, + "objective/entropy": -41.68758773803711, + "objective/kl": 35.73942184448242, + "objective/non_score_reward": -1.7869713306427002, + "objective/rlhf_reward": 15.314591407775879, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008846010314300656, + "policy/clipfrac_avg": 0.00692939106374979, + "policy/entropy_avg": 0.17317676544189453, + "step": 810, + "val/clipfrac_avg": 0.0032868448179215193, + "val/num_eos_tokens": 327, + "val/ratio": 0.9892865419387817, + "val/ratio_var": 4.3601932702586055e-05 + }, + { + "episode": 12976, + "epoch": 0.6553535353535354, + "eps": 0, + "loss/policy_avg": 0.1456170380115509, + "loss/value_avg": 0.5115708112716675, + "lr": 2.9273441165485227e-06, + "objective/entropy": -45.97090148925781, + "objective/kl": 36.01569366455078, + "objective/non_score_reward": -1.8007844686508179, + "objective/rlhf_reward": 14.23046588897705, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009762131376191974, + "policy/clipfrac_avg": 0.004803863354027271, + "policy/entropy_avg": 0.18830299377441406, + "step": 811, + "val/clipfrac_avg": 0.07713419198989868, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878898859024048, + "val/ratio_var": 4.7962184908101335e-05 + }, + { + "episode": 12992, + "epoch": 0.6561616161616162, + "eps": 0, + "loss/policy_avg": 0.1002146378159523, + "loss/value_avg": 0.558029294013977, + "lr": 2.9151960861933616e-06, + "objective/entropy": -66.14154052734375, + "objective/kl": 36.481346130371094, + "objective/non_score_reward": -1.8240673542022705, + "objective/rlhf_reward": 14.175932884216309, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009823704604059458, + "policy/clipfrac_avg": 0.0034461808390915394, + "policy/entropy_avg": 0.1955432891845703, + "step": 812, + "val/clipfrac_avg": 0.10748951137065887, + "val/num_eos_tokens": 292, + "val/ratio": 0.986059308052063, + "val/ratio_var": 5.328569022822194e-05 + }, + { + "episode": 13008, + "epoch": 0.656969696969697, + "eps": 0, + "loss/policy_avg": -0.10509809851646423, + "loss/value_avg": 0.46006062626838684, + "lr": 2.9030629340267165e-06, + "objective/entropy": -110.846923828125, + "objective/kl": 36.021034240722656, + "objective/non_score_reward": -1.8010516166687012, + "objective/rlhf_reward": 16.33176040649414, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008035163627937436, + "policy/clipfrac_avg": 0.005940265022218227, + "policy/entropy_avg": 0.15110206604003906, + "step": 813, + "val/clipfrac_avg": 0.07604548335075378, + "val/num_eos_tokens": 523, + "val/ratio": 0.9898133277893066, + "val/ratio_var": 6.0903592384420335e-05 + }, + { + "episode": 13024, + "epoch": 0.6577777777777778, + "eps": 0, + "loss/policy_avg": -0.11711955070495605, + "loss/value_avg": 1.2142528295516968, + "lr": 2.890944746636748e-06, + "objective/entropy": -40.75424575805664, + "objective/kl": 34.03726577758789, + "objective/non_score_reward": -1.701863408088684, + "objective/rlhf_reward": 13.091104507446289, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.000980376498773694, + "policy/clipfrac_avg": 0.008426186628639698, + "policy/entropy_avg": 0.20235157012939453, + "step": 814, + "val/clipfrac_avg": 0.02976190485060215, + "val/num_eos_tokens": 113, + "val/ratio": 0.9871314764022827, + "val/ratio_var": 6.27562912995927e-05 + }, + { + "episode": 13040, + "epoch": 0.6585858585858586, + "eps": 0, + "loss/policy_avg": -0.18143659830093384, + "loss/value_avg": 0.571576714515686, + "lr": 2.8788416105048124e-06, + "objective/entropy": -141.41366577148438, + "objective/kl": 42.53788757324219, + "objective/non_score_reward": -2.126894474029541, + "objective/rlhf_reward": 13.537168502807617, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008768361294642091, + "policy/clipfrac_avg": 0.006732264067977667, + "policy/entropy_avg": 0.17724037170410156, + "step": 815, + "val/clipfrac_avg": 0.03276197984814644, + "val/num_eos_tokens": 221, + "val/ratio": 0.9885480403900146, + "val/ratio_var": 8.229191007558256e-05 + }, + { + "episode": 13056, + "epoch": 0.6593939393939394, + "eps": 0, + "loss/policy_avg": 0.3017633855342865, + "loss/value_avg": 0.32543864846229553, + "lr": 2.8667536120048616e-06, + "objective/entropy": -61.10264587402344, + "objective/kl": 35.6143684387207, + "objective/non_score_reward": -1.7807183265686035, + "objective/rlhf_reward": 14.391157150268555, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009194887243211269, + "policy/clipfrac_avg": 0.018624408170580864, + "policy/entropy_avg": 0.14903783798217773, + "step": 816, + "val/clipfrac_avg": 0.0073128631338477135, + "val/num_eos_tokens": 112, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.833535033161752e-05 + }, + { + "episode": 13072, + "epoch": 0.6602020202020202, + "eps": 0, + "loss/policy_avg": 0.06899265199899673, + "loss/value_avg": 0.6989455223083496, + "lr": 2.854680837402804e-06, + "objective/entropy": -117.86632537841797, + "objective/kl": 34.15174102783203, + "objective/non_score_reward": -1.7075870037078857, + "objective/rlhf_reward": 14.511162757873535, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009505614871159196, + "policy/clipfrac_avg": 0.008451011963188648, + "policy/entropy_avg": 0.1699972152709961, + "step": 817, + "val/clipfrac_avg": 0.000589622650295496, + "val/num_eos_tokens": 435, + "val/ratio": 0.9876314401626587, + "val/ratio_var": 9.73327478277497e-05 + }, + { + "episode": 13088, + "epoch": 0.661010101010101, + "eps": 0, + "loss/policy_avg": -0.05428570508956909, + "loss/value_avg": 0.2928343415260315, + "lr": 2.8426233728559172e-06, + "objective/entropy": -52.28473663330078, + "objective/kl": 40.512550354003906, + "objective/non_score_reward": -2.025627613067627, + "objective/rlhf_reward": 14.458747863769531, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008845083648338914, + "policy/clipfrac_avg": 0.005094105843454599, + "policy/entropy_avg": 0.16685867309570312, + "step": 818, + "val/clipfrac_avg": 0.09271319210529327, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885064959526062, + "val/ratio_var": 4.853576319874264e-05 + }, + { + "episode": 13104, + "epoch": 0.6618181818181819, + "eps": 0, + "loss/policy_avg": 0.12085426598787308, + "loss/value_avg": 0.5670195817947388, + "lr": 2.83058130441221e-06, + "objective/entropy": -196.3238983154297, + "objective/kl": 42.012672424316406, + "objective/non_score_reward": -2.1006336212158203, + "objective/rlhf_reward": 12.51655387878418, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007904545636847615, + "policy/clipfrac_avg": 0.006419865880161524, + "policy/entropy_avg": 0.1437664031982422, + "step": 819, + "val/clipfrac_avg": 0.05388574302196503, + "val/num_eos_tokens": 496, + "val/ratio": 0.9899193644523621, + "val/ratio_var": 4.292546145734377e-05 + }, + { + "episode": 13120, + "epoch": 0.6626262626262627, + "eps": 0, + "loss/policy_avg": 0.6058844327926636, + "loss/value_avg": 0.7294787168502808, + "lr": 2.8185547180098195e-06, + "objective/entropy": -58.513553619384766, + "objective/kl": 38.73785400390625, + "objective/non_score_reward": -1.9368927478790283, + "objective/rlhf_reward": 12.883419036865234, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010288474150002003, + "policy/clipfrac_avg": 0.021498143672943115, + "policy/entropy_avg": 0.20896148681640625, + "step": 820, + "val/clipfrac_avg": 0.0723545029759407, + "val/num_eos_tokens": 134, + "val/ratio": 0.9887969493865967, + "val/ratio_var": 4.8135509132407606e-05 + }, + { + "episode": 13136, + "epoch": 0.6634343434343435, + "eps": 0, + "loss/policy_avg": -0.21932660043239594, + "loss/value_avg": 0.45023149251937866, + "lr": 2.8065436994763963e-06, + "objective/entropy": -38.43750762939453, + "objective/kl": 33.36050033569336, + "objective/non_score_reward": -1.6680251359939575, + "objective/rlhf_reward": 13.542912483215332, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010698122205212712, + "policy/clipfrac_avg": 0.004931025672703981, + "policy/entropy_avg": 0.19058465957641602, + "step": 821, + "val/clipfrac_avg": 0.00852490495890379, + "val/num_eos_tokens": 190, + "val/ratio": 0.9864101409912109, + "val/ratio_var": 5.521172352018766e-05 + }, + { + "episode": 13152, + "epoch": 0.6642424242424242, + "eps": 0, + "loss/policy_avg": -0.01703064516186714, + "loss/value_avg": 0.2902008295059204, + "lr": 2.794548334528486e-06, + "objective/entropy": 13.10763931274414, + "objective/kl": 44.57916259765625, + "objective/non_score_reward": -2.2289581298828125, + "objective/rlhf_reward": 13.364792823791504, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0014038155786693096, + "policy/clipfrac_avg": 0.006509849801659584, + "policy/entropy_avg": 0.26366424560546875, + "step": 822, + "val/clipfrac_avg": 0.03128276392817497, + "val/num_eos_tokens": 35, + "val/ratio": 0.9824488162994385, + "val/ratio_var": 4.7705398174002767e-05 + }, + { + "episode": 13168, + "epoch": 0.665050505050505, + "eps": 0, + "loss/policy_avg": -0.02819274365901947, + "loss/value_avg": 0.2861873507499695, + "lr": 2.782568708770933e-06, + "objective/entropy": -94.84840393066406, + "objective/kl": 42.75294494628906, + "objective/non_score_reward": -2.1376476287841797, + "objective/rlhf_reward": 12.26860237121582, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010491260327398777, + "policy/clipfrac_avg": 0.004985281266272068, + "policy/entropy_avg": 0.2000885009765625, + "step": 823, + "val/clipfrac_avg": 0.00754884909838438, + "val/num_eos_tokens": 289, + "val/ratio": 0.9866741895675659, + "val/ratio_var": 4.918476770399138e-05 + }, + { + "episode": 13184, + "epoch": 0.6658585858585858, + "eps": 0, + "loss/policy_avg": -0.024818487465381622, + "loss/value_avg": 0.3611629605293274, + "lr": 2.7706049076962483e-06, + "objective/entropy": -104.7535400390625, + "objective/kl": 45.08838653564453, + "objective/non_score_reward": -2.2544193267822266, + "objective/rlhf_reward": 12.214330673217773, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009155337465927005, + "policy/clipfrac_avg": 0.008831838145852089, + "policy/entropy_avg": 0.16748523712158203, + "step": 824, + "val/clipfrac_avg": 0.0024258445482701063, + "val/num_eos_tokens": 163, + "val/ratio": 0.9890375137329102, + "val/ratio_var": 5.131318175699562e-05 + }, + { + "episode": 13200, + "epoch": 0.6666666666666666, + "eps": 0, + "loss/policy_avg": -0.28224268555641174, + "loss/value_avg": 0.8720104694366455, + "lr": 2.7586570166840154e-06, + "objective/entropy": -74.55836486816406, + "objective/kl": 30.59856605529785, + "objective/non_score_reward": -1.52992844581604, + "objective/rlhf_reward": 15.329446792602539, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008330290438607335, + "policy/clipfrac_avg": 0.010464916005730629, + "policy/entropy_avg": 0.15100550651550293, + "step": 825, + "val/clipfrac_avg": 0.08580771088600159, + "val/num_eos_tokens": 460, + "val/ratio": 0.9896235466003418, + "val/ratio_var": 9.328931628260761e-05 + }, + { + "episode": 13216, + "epoch": 0.6674747474747474, + "eps": 0, + "loss/policy_avg": -0.1715128868818283, + "loss/value_avg": 0.4540337920188904, + "lr": 2.746725121000273e-06, + "objective/entropy": -43.89616394042969, + "objective/kl": 41.760223388671875, + "objective/non_score_reward": -2.0880112648010254, + "objective/rlhf_reward": 13.872926712036133, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010317859705537558, + "policy/clipfrac_avg": 0.0054151020012795925, + "policy/entropy_avg": 0.19500446319580078, + "step": 826, + "val/clipfrac_avg": 0.03631855547428131, + "val/num_eos_tokens": 334, + "val/ratio": 0.9874616265296936, + "val/ratio_var": 6.214805034687743e-05 + }, + { + "episode": 13232, + "epoch": 0.6682828282828283, + "eps": 0, + "loss/policy_avg": -0.077702596783638, + "loss/value_avg": 0.6576616168022156, + "lr": 2.734809305796915e-06, + "objective/entropy": -13.071683883666992, + "objective/kl": 34.05264663696289, + "objective/non_score_reward": -1.7026324272155762, + "objective/rlhf_reward": 14.656742095947266, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0012533586705103517, + "policy/clipfrac_avg": 0.00517519935965538, + "policy/entropy_avg": 0.21735763549804688, + "step": 827, + "val/clipfrac_avg": 0.09026823937892914, + "val/num_eos_tokens": 468, + "val/ratio": 0.9863014221191406, + "val/ratio_var": 0.00010488701082067564 + }, + { + "episode": 13248, + "epoch": 0.6690909090909091, + "eps": 0, + "loss/policy_avg": -0.11604561656713486, + "loss/value_avg": 0.18491807579994202, + "lr": 2.7229096561110703e-06, + "objective/entropy": -132.30560302734375, + "objective/kl": 37.0118522644043, + "objective/non_score_reward": -1.8505926132202148, + "objective/rlhf_reward": 15.282218933105469, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008336640894412994, + "policy/clipfrac_avg": 0.0030343476682901382, + "policy/entropy_avg": 0.16221046447753906, + "step": 828, + "val/clipfrac_avg": 0.07020965218544006, + "val/num_eos_tokens": 41, + "val/ratio": 0.9894965291023254, + "val/ratio_var": 4.505051037995145e-05 + }, + { + "episode": 13264, + "epoch": 0.6698989898989899, + "eps": 0, + "loss/policy_avg": -0.23082350194454193, + "loss/value_avg": 0.9338909387588501, + "lr": 2.7110262568645063e-06, + "objective/entropy": -83.5711441040039, + "objective/kl": 42.80398941040039, + "objective/non_score_reward": -2.1401994228363037, + "objective/rlhf_reward": 13.555112838745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008490884792990983, + "policy/clipfrac_avg": 0.024211544543504715, + "policy/entropy_avg": 0.16208171844482422, + "step": 829, + "val/clipfrac_avg": 0.01703277789056301, + "val/num_eos_tokens": 407, + "val/ratio": 0.9889751672744751, + "val/ratio_var": 6.549843965331092e-05 + }, + { + "episode": 13280, + "epoch": 0.6707070707070707, + "eps": 0, + "loss/policy_avg": -0.1790890097618103, + "loss/value_avg": 0.9402360320091248, + "lr": 2.699159192863018e-06, + "objective/entropy": -91.52725219726562, + "objective/kl": 30.28129005432129, + "objective/non_score_reward": -1.5140645503997803, + "objective/rlhf_reward": 13.71249771118164, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0006465939804911613, + "policy/clipfrac_avg": 0.003674568608403206, + "policy/entropy_avg": 0.14063799381256104, + "step": 830, + "val/clipfrac_avg": 0.05747377499938011, + "val/num_eos_tokens": 250, + "val/ratio": 0.9919025897979736, + "val/ratio_var": 6.961268809391186e-05 + }, + { + "episode": 13296, + "epoch": 0.6715151515151515, + "eps": 0, + "loss/policy_avg": 0.3400639295578003, + "loss/value_avg": 1.9688762426376343, + "lr": 2.687308548795825e-06, + "objective/entropy": -54.39606475830078, + "objective/kl": 44.12242889404297, + "objective/non_score_reward": -2.2061214447021484, + "objective/rlhf_reward": 14.004816055297852, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009418983245268464, + "policy/clipfrac_avg": 0.009523827582597733, + "policy/entropy_avg": 0.18965661525726318, + "step": 831, + "val/clipfrac_avg": 0.045346200466156006, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880996942520142, + "val/ratio_var": 6.565819057868794e-05 + }, + { + "episode": 13312, + "epoch": 0.6723232323232323, + "eps": 0, + "loss/policy_avg": 0.11609108746051788, + "loss/value_avg": 0.5635161399841309, + "lr": 2.675474409234965e-06, + "objective/entropy": -66.36386108398438, + "objective/kl": 38.93806838989258, + "objective/non_score_reward": -1.9469034671783447, + "objective/rlhf_reward": 14.076534271240234, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008638559957034886, + "policy/clipfrac_avg": 0.005322727840393782, + "policy/entropy_avg": 0.15152549743652344, + "step": 832, + "val/clipfrac_avg": 0.04381619766354561, + "val/num_eos_tokens": 20, + "val/ratio": 0.9889363050460815, + "val/ratio_var": 3.002676021424122e-05 + }, + { + "episode": 13328, + "epoch": 0.6731313131313131, + "eps": 0, + "loss/policy_avg": -0.03308400511741638, + "loss/value_avg": 0.3210276961326599, + "lr": 2.66365685863469e-06, + "objective/entropy": -111.39958953857422, + "objective/kl": 47.16798400878906, + "objective/non_score_reward": -2.3583993911743164, + "objective/rlhf_reward": 11.7197265625, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009453936363570392, + "policy/clipfrac_avg": 0.004603059496730566, + "policy/entropy_avg": 0.18264532089233398, + "step": 833, + "val/clipfrac_avg": 0.062541663646698, + "val/num_eos_tokens": 763, + "val/ratio": 0.9879401922225952, + "val/ratio_var": 6.487595965154469e-05 + }, + { + "episode": 13344, + "epoch": 0.673939393939394, + "eps": 0, + "loss/policy_avg": 0.051825061440467834, + "loss/value_avg": 0.8246160745620728, + "lr": 2.651855981330872e-06, + "objective/entropy": -69.66492462158203, + "objective/kl": 37.68748474121094, + "objective/non_score_reward": -1.8843741416931152, + "objective/rlhf_reward": 12.764063835144043, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010055399034172297, + "policy/clipfrac_avg": 0.004349449649453163, + "policy/entropy_avg": 0.18214893341064453, + "step": 834, + "val/clipfrac_avg": 0.06374436616897583, + "val/num_eos_tokens": 38, + "val/ratio": 0.9867125749588013, + "val/ratio_var": 9.083253826247528e-05 + }, + { + "episode": 13360, + "epoch": 0.6747474747474748, + "eps": 0, + "loss/policy_avg": -0.043558381497859955, + "loss/value_avg": 0.6093795895576477, + "lr": 2.6400718615403852e-06, + "objective/entropy": -108.79554748535156, + "objective/kl": 39.288482666015625, + "objective/non_score_reward": -1.9644241333007812, + "objective/rlhf_reward": 14.504325866699219, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010659170802682638, + "policy/clipfrac_avg": 0.005707671865820885, + "policy/entropy_avg": 0.1986398696899414, + "step": 835, + "val/clipfrac_avg": 0.013371335342526436, + "val/num_eos_tokens": 647, + "val/ratio": 0.9869523048400879, + "val/ratio_var": 8.708865061635152e-05 + }, + { + "episode": 13376, + "epoch": 0.6755555555555556, + "eps": 0, + "loss/policy_avg": 0.09040576219558716, + "loss/value_avg": 0.4517725706100464, + "lr": 2.62830458336052e-06, + "objective/entropy": -59.1908073425293, + "objective/kl": 42.06044387817383, + "objective/non_score_reward": -2.1030220985412598, + "objective/rlhf_reward": 13.842290878295898, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001016529044136405, + "policy/clipfrac_avg": 0.007354079745709896, + "policy/entropy_avg": 0.20728302001953125, + "step": 836, + "val/clipfrac_avg": 0.030187882483005524, + "val/num_eos_tokens": 477, + "val/ratio": 0.9865027666091919, + "val/ratio_var": 5.405073534348048e-05 + }, + { + "episode": 13392, + "epoch": 0.6763636363636364, + "eps": 0, + "loss/policy_avg": -0.21371892094612122, + "loss/value_avg": 0.26992350816726685, + "lr": 2.6165542307683744e-06, + "objective/entropy": -99.01640319824219, + "objective/kl": 43.636474609375, + "objective/non_score_reward": -2.18182373046875, + "objective/rlhf_reward": 13.08380126953125, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009176525054499507, + "policy/clipfrac_avg": 0.006078221835196018, + "policy/entropy_avg": 0.1714763641357422, + "step": 837, + "val/clipfrac_avg": 0.025651777163147926, + "val/num_eos_tokens": 453, + "val/ratio": 0.988213300704956, + "val/ratio_var": 6.253684114199132e-05 + }, + { + "episode": 13408, + "epoch": 0.6771717171717172, + "eps": 0, + "loss/policy_avg": 0.08225658535957336, + "loss/value_avg": 0.4106031656265259, + "lr": 2.604820887620254e-06, + "objective/entropy": -62.21269989013672, + "objective/kl": 40.305843353271484, + "objective/non_score_reward": -2.015292167663574, + "objective/rlhf_reward": 15.75814437866211, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0009111049002967775, + "policy/clipfrac_avg": 0.005017759278416634, + "policy/entropy_avg": 0.18220901489257812, + "step": 838, + "val/clipfrac_avg": 0.0028512999415397644, + "val/num_eos_tokens": 268, + "val/ratio": 0.9878619909286499, + "val/ratio_var": 3.237010969314724e-05 + }, + { + "episode": 13424, + "epoch": 0.677979797979798, + "eps": 0, + "loss/policy_avg": 0.01355072483420372, + "loss/value_avg": 0.4876829981803894, + "lr": 2.5931046376510875e-06, + "objective/entropy": -38.304931640625, + "objective/kl": 35.24858093261719, + "objective/non_score_reward": -1.7624289989471436, + "objective/rlhf_reward": 12.284445762634277, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0008708706009201705, + "policy/clipfrac_avg": 0.005640837829560041, + "policy/entropy_avg": 0.16802191734313965, + "step": 839, + "val/clipfrac_avg": 0.029899097979068756, + "val/num_eos_tokens": 258, + "val/ratio": 0.989201009273529, + "val/ratio_var": 5.1689257816178724e-05 + }, + { + "episode": 13440, + "epoch": 0.6787878787878788, + "eps": 0, + "loss/policy_avg": 0.1623387336730957, + "loss/value_avg": 0.3584904074668884, + "lr": 2.5814055644738013e-06, + "objective/entropy": -63.923397064208984, + "objective/kl": 36.323509216308594, + "objective/non_score_reward": -1.8161752223968506, + "objective/rlhf_reward": 12.97288703918457, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009589061955921352, + "policy/clipfrac_avg": 0.006033012643456459, + "policy/entropy_avg": 0.17229270935058594, + "step": 840, + "val/clipfrac_avg": 0.11569125950336456, + "val/num_eos_tokens": 159, + "val/ratio": 0.988390326499939, + "val/ratio_var": 7.233954966068268e-05 + }, + { + "episode": 13456, + "epoch": 0.6795959595959596, + "eps": 0, + "loss/policy_avg": -0.29571330547332764, + "loss/value_avg": 0.5451829433441162, + "lr": 2.5697237515787564e-06, + "objective/entropy": -159.0351104736328, + "objective/kl": 34.90948486328125, + "objective/non_score_reward": -1.7454744577407837, + "objective/rlhf_reward": 13.145150184631348, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0007017417810857296, + "policy/clipfrac_avg": 0.005396063905209303, + "policy/entropy_avg": 0.12675732374191284, + "step": 841, + "val/clipfrac_avg": 0.019056372344493866, + "val/num_eos_tokens": 508, + "val/ratio": 0.9914215207099915, + "val/ratio_var": 8.740482735447586e-05 + }, + { + "episode": 13472, + "epoch": 0.6804040404040405, + "eps": 0, + "loss/policy_avg": -0.33980700373649597, + "loss/value_avg": 0.5796597003936768, + "lr": 2.5580592823331237e-06, + "objective/entropy": -83.5593490600586, + "objective/kl": 36.281490325927734, + "objective/non_score_reward": -1.8140745162963867, + "objective/rlhf_reward": 14.18592643737793, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009474118705838919, + "policy/clipfrac_avg": 0.004876702558249235, + "policy/entropy_avg": 0.14034080505371094, + "step": 842, + "val/clipfrac_avg": 0.03151373565196991, + "val/num_eos_tokens": 208, + "val/ratio": 0.9901444911956787, + "val/ratio_var": 6.344315625028685e-05 + }, + { + "episode": 13488, + "epoch": 0.6812121212121212, + "eps": 0, + "loss/policy_avg": 0.11847849935293198, + "loss/value_avg": 0.3053319454193115, + "lr": 2.5464122399803126e-06, + "objective/entropy": -74.54005432128906, + "objective/kl": 46.23011779785156, + "objective/non_score_reward": -2.3115057945251465, + "objective/rlhf_reward": 12.993181228637695, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009884861065074801, + "policy/clipfrac_avg": 0.006667924579232931, + "policy/entropy_avg": 0.19658470153808594, + "step": 843, + "val/clipfrac_avg": 0.00010080645006382838, + "val/num_eos_tokens": 52, + "val/ratio": 0.9878820180892944, + "val/ratio_var": 4.166604412603192e-05 + }, + { + "episode": 13504, + "epoch": 0.682020202020202, + "eps": 0, + "loss/policy_avg": 0.07477365434169769, + "loss/value_avg": 0.717211902141571, + "lr": 2.5347827076393505e-06, + "objective/entropy": -55.136016845703125, + "objective/kl": 36.799835205078125, + "objective/non_score_reward": -1.8399916887283325, + "objective/rlhf_reward": 12.995945930480957, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010256943060085177, + "policy/clipfrac_avg": 0.009538116864860058, + "policy/entropy_avg": 0.19795989990234375, + "step": 844, + "val/clipfrac_avg": 0.0769520252943039, + "val/num_eos_tokens": 573, + "val/ratio": 0.9861817955970764, + "val/ratio_var": 7.031750283204019e-05 + }, + { + "episode": 13520, + "epoch": 0.6828282828282828, + "eps": 0, + "loss/policy_avg": -0.010380454361438751, + "loss/value_avg": 0.8978620767593384, + "lr": 2.5231707683043204e-06, + "objective/entropy": -164.994873046875, + "objective/kl": 41.09418487548828, + "objective/non_score_reward": -2.0547094345092773, + "objective/rlhf_reward": 12.367165565490723, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006862258305773139, + "policy/clipfrac_avg": 0.005633420776575804, + "policy/entropy_avg": 0.136183500289917, + "step": 845, + "val/clipfrac_avg": 0.10761284828186035, + "val/num_eos_tokens": 289, + "val/ratio": 0.9907996654510498, + "val/ratio_var": 4.169603198533878e-05 + }, + { + "episode": 13536, + "epoch": 0.6836363636363636, + "eps": 0, + "loss/policy_avg": -0.0011942088603973389, + "loss/value_avg": 0.7805696129798889, + "lr": 2.5115765048437445e-06, + "objective/entropy": -119.26876068115234, + "objective/kl": 37.73551940917969, + "objective/non_score_reward": -1.8867759704589844, + "objective/rlhf_reward": 13.769474029541016, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0007093651220202446, + "policy/clipfrac_avg": 0.028275154531002045, + "policy/entropy_avg": 0.1421506404876709, + "step": 846, + "val/clipfrac_avg": 0.08967090398073196, + "val/num_eos_tokens": 760, + "val/ratio": 0.9911968111991882, + "val/ratio_var": 3.498927253531292e-05 + }, + { + "episode": 13552, + "epoch": 0.6844444444444444, + "eps": 0, + "loss/policy_avg": -0.007019490003585815, + "loss/value_avg": 0.8198627233505249, + "lr": 2.5000000000000015e-06, + "objective/entropy": -89.728271484375, + "objective/kl": 34.76280975341797, + "objective/non_score_reward": -1.7381404638290405, + "objective/rlhf_reward": 13.925922393798828, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009024562314152718, + "policy/clipfrac_avg": 0.007501127198338509, + "policy/entropy_avg": 0.17042064666748047, + "step": 847, + "val/clipfrac_avg": 0.028372831642627716, + "val/num_eos_tokens": 17, + "val/ratio": 0.9899512529373169, + "val/ratio_var": 7.211479532998055e-05 + }, + { + "episode": 13568, + "epoch": 0.6852525252525252, + "eps": 0, + "loss/policy_avg": 0.015100933611392975, + "loss/value_avg": 0.25284868478775024, + "lr": 2.488441336388737e-06, + "objective/entropy": -184.20425415039062, + "objective/kl": 46.20475769042969, + "objective/non_score_reward": -2.3102383613586426, + "objective/rlhf_reward": 11.728824615478516, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007638686802238226, + "policy/clipfrac_avg": 0.005230149254202843, + "policy/entropy_avg": 0.15384864807128906, + "step": 848, + "val/clipfrac_avg": 0.01555095985531807, + "val/num_eos_tokens": 404, + "val/ratio": 0.9898365139961243, + "val/ratio_var": 5.412843165686354e-05 + }, + { + "episode": 13584, + "epoch": 0.686060606060606, + "eps": 0, + "loss/policy_avg": -0.13074953854084015, + "loss/value_avg": 0.5254482626914978, + "lr": 2.4769005964982718e-06, + "objective/entropy": -70.96700286865234, + "objective/kl": 37.31745147705078, + "objective/non_score_reward": -1.8658727407455444, + "objective/rlhf_reward": 13.665377616882324, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000885509536601603, + "policy/clipfrac_avg": 0.006637534126639366, + "policy/entropy_avg": 0.17204856872558594, + "step": 849, + "val/clipfrac_avg": 0.050514861941337585, + "val/num_eos_tokens": 24, + "val/ratio": 0.9882212281227112, + "val/ratio_var": 5.447021248983219e-05 + }, + { + "episode": 13600, + "epoch": 0.6868686868686869, + "eps": 0, + "loss/policy_avg": 0.039310187101364136, + "loss/value_avg": 0.18585552275180817, + "lr": 2.465377862689019e-06, + "objective/entropy": -62.67339324951172, + "objective/kl": 48.39326477050781, + "objective/non_score_reward": -2.419663190841675, + "objective/rlhf_reward": 13.564711570739746, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009431451326236129, + "policy/clipfrac_avg": 0.004864457994699478, + "policy/entropy_avg": 0.19204330444335938, + "step": 850, + "val/clipfrac_avg": 0.01945144683122635, + "val/num_eos_tokens": 36, + "val/ratio": 0.9872653484344482, + "val/ratio_var": 3.194732926203869e-05 + }, + { + "episode": 13616, + "epoch": 0.6876767676767677, + "eps": 0, + "loss/policy_avg": 0.07975710183382034, + "loss/value_avg": 0.478973388671875, + "lr": 2.4538732171928847e-06, + "objective/entropy": -76.03438568115234, + "objective/kl": 35.364593505859375, + "objective/non_score_reward": -1.7682294845581055, + "objective/rlhf_reward": 15.075519561767578, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008544789161533117, + "policy/clipfrac_avg": 0.005153194069862366, + "policy/entropy_avg": 0.16641998291015625, + "step": 851, + "val/clipfrac_avg": 0.03764945641160011, + "val/num_eos_tokens": 298, + "val/ratio": 0.9884979724884033, + "val/ratio_var": 4.7100515075726435e-05 + }, + { + "episode": 13632, + "epoch": 0.6884848484848485, + "eps": 0, + "loss/policy_avg": -0.10211583226919174, + "loss/value_avg": 0.7199238538742065, + "lr": 2.4423867421126923e-06, + "objective/entropy": -59.744590759277344, + "objective/kl": 35.15203857421875, + "objective/non_score_reward": -1.7576020956039429, + "objective/rlhf_reward": 15.05489730834961, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009704260155558586, + "policy/clipfrac_avg": 0.004688786808401346, + "policy/entropy_avg": 0.17905235290527344, + "step": 852, + "val/clipfrac_avg": 0.17230291664600372, + "val/num_eos_tokens": 644, + "val/ratio": 0.9872374534606934, + "val/ratio_var": 4.29930878453888e-05 + }, + { + "episode": 13648, + "epoch": 0.6892929292929293, + "eps": 0, + "loss/policy_avg": 0.0028443392366170883, + "loss/value_avg": 0.3421381115913391, + "lr": 2.4309185194215885e-06, + "objective/entropy": -63.859352111816406, + "objective/kl": 41.87160110473633, + "objective/non_score_reward": -2.0935800075531006, + "objective/rlhf_reward": 16.01579475402832, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0011341955978423357, + "policy/clipfrac_avg": 0.005558031145483255, + "policy/entropy_avg": 0.21439552307128906, + "step": 853, + "val/clipfrac_avg": 0.07718860357999802, + "val/num_eos_tokens": 400, + "val/ratio": 0.9858847856521606, + "val/ratio_var": 5.448433512356132e-05 + }, + { + "episode": 13664, + "epoch": 0.6901010101010101, + "eps": 0, + "loss/policy_avg": 0.11498575657606125, + "loss/value_avg": 0.909265398979187, + "lr": 2.4194686309624664e-06, + "objective/entropy": -138.73922729492188, + "objective/kl": 39.14732360839844, + "objective/non_score_reward": -1.9573662281036377, + "objective/rlhf_reward": 11.542634010314941, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0011413603788241744, + "policy/clipfrac_avg": 0.008165586739778519, + "policy/entropy_avg": 0.18193864822387695, + "step": 854, + "val/clipfrac_avg": 0.04511754959821701, + "val/num_eos_tokens": 264, + "val/ratio": 0.9880124926567078, + "val/ratio_var": 7.85917291068472e-05 + }, + { + "episode": 13680, + "epoch": 0.6909090909090909, + "eps": 0, + "loss/policy_avg": 0.4261026978492737, + "loss/value_avg": 1.0351085662841797, + "lr": 2.408037158447375e-06, + "objective/entropy": -80.8604736328125, + "objective/kl": 39.87467956542969, + "objective/non_score_reward": -1.9937340021133423, + "objective/rlhf_reward": 12.850015640258789, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000854466634336859, + "policy/clipfrac_avg": 0.0031361770816147327, + "policy/entropy_avg": 0.1807117462158203, + "step": 855, + "val/clipfrac_avg": 0.06461638957262039, + "val/num_eos_tokens": 76, + "val/ratio": 0.9892903566360474, + "val/ratio_var": 6.158453470561653e-05 + }, + { + "episode": 13696, + "epoch": 0.6917171717171717, + "eps": 0, + "loss/policy_avg": 0.35515934228897095, + "loss/value_avg": 0.6004530787467957, + "lr": 2.3966241834569364e-06, + "objective/entropy": -109.00687408447266, + "objective/kl": 44.59175109863281, + "objective/non_score_reward": -2.2295875549316406, + "objective/rlhf_reward": 14.02041244506836, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008301680209115148, + "policy/clipfrac_avg": 0.004295141436159611, + "policy/entropy_avg": 0.16374778747558594, + "step": 856, + "val/clipfrac_avg": 0.0578775517642498, + "val/num_eos_tokens": 355, + "val/ratio": 0.9892109632492065, + "val/ratio_var": 3.649172867881134e-05 + }, + { + "episode": 13712, + "epoch": 0.6925252525252525, + "eps": 0, + "loss/policy_avg": -0.07683120667934418, + "loss/value_avg": 0.7507062554359436, + "lr": 2.385229787439768e-06, + "objective/entropy": -65.75347137451172, + "objective/kl": 33.233463287353516, + "objective/non_score_reward": -1.6616731882095337, + "objective/rlhf_reward": 12.885202407836914, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008445363491773605, + "policy/clipfrac_avg": 0.0028887931257486343, + "policy/entropy_avg": 0.1688997745513916, + "step": 857, + "val/clipfrac_avg": 0.029584139585494995, + "val/num_eos_tokens": 79, + "val/ratio": 0.9893004894256592, + "val/ratio_var": 4.4566000724444166e-05 + }, + { + "episode": 13728, + "epoch": 0.6933333333333334, + "eps": 0, + "loss/policy_avg": -0.04102202132344246, + "loss/value_avg": 0.295141339302063, + "lr": 2.3738540517118953e-06, + "objective/entropy": -137.21546936035156, + "objective/kl": 39.995941162109375, + "objective/non_score_reward": -1.9997971057891846, + "objective/rlhf_reward": 12.875202178955078, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009317261283285916, + "policy/clipfrac_avg": 0.016368409618735313, + "policy/entropy_avg": 0.18145465850830078, + "step": 858, + "val/clipfrac_avg": 0.02603679709136486, + "val/num_eos_tokens": 133, + "val/ratio": 0.9879333972930908, + "val/ratio_var": 7.448039104929194e-05 + }, + { + "episode": 13744, + "epoch": 0.6941414141414142, + "eps": 0, + "loss/policy_avg": -0.09325989335775375, + "loss/value_avg": 0.4778023660182953, + "lr": 2.3624970574561773e-06, + "objective/entropy": -126.39140319824219, + "objective/kl": 41.33527755737305, + "objective/non_score_reward": -2.0667638778686523, + "objective/rlhf_reward": 15.823860168457031, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009032993111759424, + "policy/clipfrac_avg": 0.008262147195637226, + "policy/entropy_avg": 0.1809234619140625, + "step": 859, + "val/clipfrac_avg": 0.006358179729431868, + "val/num_eos_tokens": 198, + "val/ratio": 0.9881711602210999, + "val/ratio_var": 8.430143498117104e-05 + }, + { + "episode": 13760, + "epoch": 0.694949494949495, + "eps": 0, + "loss/policy_avg": -0.1992878019809723, + "loss/value_avg": 0.4450816214084625, + "lr": 2.3511588857217228e-06, + "objective/entropy": -82.32437133789062, + "objective/kl": 36.61549377441406, + "objective/non_score_reward": -1.8307745456695557, + "objective/rlhf_reward": 14.497350692749023, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009405683958902955, + "policy/clipfrac_avg": 0.0031182412058115005, + "policy/entropy_avg": 0.1852397918701172, + "step": 860, + "val/clipfrac_avg": 0.04030741751194, + "val/num_eos_tokens": 674, + "val/ratio": 0.9874944686889648, + "val/ratio_var": 5.9595575294224545e-05 + }, + { + "episode": 13776, + "epoch": 0.6957575757575758, + "eps": 0, + "loss/policy_avg": -0.0025226399302482605, + "loss/value_avg": 0.6712653636932373, + "lr": 2.339839617423318e-06, + "objective/entropy": -97.76516723632812, + "objective/kl": 32.36253356933594, + "objective/non_score_reward": -1.6181267499923706, + "objective/rlhf_reward": 14.108436584472656, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008775125024840236, + "policy/clipfrac_avg": 0.006763250567018986, + "policy/entropy_avg": 0.15996170043945312, + "step": 861, + "val/clipfrac_avg": 0.01572282984852791, + "val/num_eos_tokens": 755, + "val/ratio": 0.9897935390472412, + "val/ratio_var": 4.971764428773895e-05 + }, + { + "episode": 13792, + "epoch": 0.6965656565656566, + "eps": 0, + "loss/policy_avg": 0.08292508125305176, + "loss/value_avg": 0.5537270307540894, + "lr": 2.32853933334084e-06, + "objective/entropy": -61.261680603027344, + "objective/kl": 36.61561584472656, + "objective/non_score_reward": -1.8307808637619019, + "objective/rlhf_reward": 12.762969970703125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009761417168192565, + "policy/clipfrac_avg": 0.0026826506946235895, + "policy/entropy_avg": 0.22123146057128906, + "step": 862, + "val/clipfrac_avg": 0.044523607939481735, + "val/num_eos_tokens": 56, + "val/ratio": 0.9871484041213989, + "val/ratio_var": 7.335093687288463e-05 + }, + { + "episode": 13808, + "epoch": 0.6973737373737374, + "eps": 0, + "loss/policy_avg": 0.36910420656204224, + "loss/value_avg": 0.9228214025497437, + "lr": 2.317258114118686e-06, + "objective/entropy": -76.75859069824219, + "objective/kl": 36.92033004760742, + "objective/non_score_reward": -1.8460166454315186, + "objective/rlhf_reward": 14.872733116149902, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009935328271239996, + "policy/clipfrac_avg": 0.006773949135094881, + "policy/entropy_avg": 0.1806344985961914, + "step": 863, + "val/clipfrac_avg": 0.07717607915401459, + "val/num_eos_tokens": 514, + "val/ratio": 0.9876723289489746, + "val/ratio_var": 6.279607623582706e-05 + }, + { + "episode": 13824, + "epoch": 0.6981818181818182, + "eps": 0, + "loss/policy_avg": -0.21535149216651917, + "loss/value_avg": 0.42513978481292725, + "lr": 2.305996040265198e-06, + "objective/entropy": -77.51583862304688, + "objective/kl": 45.520545959472656, + "objective/non_score_reward": -2.276027202606201, + "objective/rlhf_reward": 14.59897232055664, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008458065567538142, + "policy/clipfrac_avg": 0.005439429543912411, + "policy/entropy_avg": 0.16998863220214844, + "step": 864, + "val/clipfrac_avg": 0.03808614984154701, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894691109657288, + "val/ratio_var": 6.017706618877128e-05 + }, + { + "episode": 13840, + "epoch": 0.6989898989898989, + "eps": 0, + "loss/policy_avg": -0.08341407775878906, + "loss/value_avg": 0.7828294038772583, + "lr": 2.29475319215209e-06, + "objective/entropy": -82.54826354980469, + "objective/kl": 39.21794891357422, + "objective/non_score_reward": -1.9608973264694214, + "objective/rlhf_reward": 14.359415054321289, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007373163243755698, + "policy/clipfrac_avg": 0.003970882389694452, + "policy/entropy_avg": 0.14882469177246094, + "step": 865, + "val/clipfrac_avg": 0.17675819993019104, + "val/num_eos_tokens": 134, + "val/ratio": 0.9905297756195068, + "val/ratio_var": 5.841355596203357e-05 + }, + { + "episode": 13856, + "epoch": 0.6997979797979798, + "eps": 0, + "loss/policy_avg": 0.029253050684928894, + "loss/value_avg": 0.8540998101234436, + "lr": 2.2835296500138664e-06, + "objective/entropy": -121.09165954589844, + "objective/kl": 34.1153564453125, + "objective/non_score_reward": -1.7057678699493408, + "objective/rlhf_reward": 13.333294868469238, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009264532127417624, + "policy/clipfrac_avg": 0.0050398847088217735, + "policy/entropy_avg": 0.18099594116210938, + "step": 866, + "val/clipfrac_avg": 0.02329857461154461, + "val/num_eos_tokens": 1353, + "val/ratio": 0.9883856177330017, + "val/ratio_var": 8.088548202067614e-05 + }, + { + "episode": 13872, + "epoch": 0.7006060606060606, + "eps": 0, + "loss/policy_avg": -0.1498204916715622, + "loss/value_avg": 0.4047302007675171, + "lr": 2.272325493947257e-06, + "objective/entropy": -49.65830993652344, + "objective/kl": 36.36531066894531, + "objective/non_score_reward": -1.8182655572891235, + "objective/rlhf_reward": 14.494234085083008, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010429773246869445, + "policy/clipfrac_avg": 0.005474097095429897, + "policy/entropy_avg": 0.18477344512939453, + "step": 867, + "val/clipfrac_avg": 0.0032526040449738503, + "val/num_eos_tokens": 101, + "val/ratio": 0.9877700805664062, + "val/ratio_var": 9.155944280792028e-05 + }, + { + "episode": 13888, + "epoch": 0.7014141414141414, + "eps": 0, + "loss/policy_avg": -0.1165279895067215, + "loss/value_avg": 0.7669879198074341, + "lr": 2.261140803910644e-06, + "objective/entropy": -77.34773254394531, + "objective/kl": 38.648643493652344, + "objective/non_score_reward": -1.9324320554733276, + "objective/rlhf_reward": 13.637880325317383, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000937658129259944, + "policy/clipfrac_avg": 0.006235727574676275, + "policy/entropy_avg": 0.19645977020263672, + "step": 868, + "val/clipfrac_avg": 0.058441728353500366, + "val/num_eos_tokens": 310, + "val/ratio": 0.9879204034805298, + "val/ratio_var": 6.414015660993755e-05 + }, + { + "episode": 13904, + "epoch": 0.7022222222222222, + "eps": 0, + "loss/policy_avg": -0.016327757388353348, + "loss/value_avg": 0.43844515085220337, + "lr": 2.2499756597234872e-06, + "objective/entropy": -163.28765869140625, + "objective/kl": 48.81157684326172, + "objective/non_score_reward": -2.4405789375305176, + "objective/rlhf_reward": 11.70395278930664, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.000844982685521245, + "policy/clipfrac_avg": 0.0032518147490918636, + "policy/entropy_avg": 0.16161727905273438, + "step": 869, + "val/clipfrac_avg": 0.014801801182329655, + "val/num_eos_tokens": 1949, + "val/ratio": 0.9885438680648804, + "val/ratio_var": 5.214377597440034e-05 + }, + { + "episode": 13920, + "epoch": 0.703030303030303, + "eps": 0, + "loss/policy_avg": -0.18964827060699463, + "loss/value_avg": 0.5136280655860901, + "lr": 2.238830141065765e-06, + "objective/entropy": -90.82677459716797, + "objective/kl": 40.3735237121582, + "objective/non_score_reward": -2.0186760425567627, + "objective/rlhf_reward": 15.153199195861816, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008435281342826784, + "policy/clipfrac_avg": 0.005789198447018862, + "policy/entropy_avg": 0.1680126190185547, + "step": 870, + "val/clipfrac_avg": 0.0648677870631218, + "val/num_eos_tokens": 555, + "val/ratio": 0.9890276193618774, + "val/ratio_var": 7.125848060240969e-05 + }, + { + "episode": 13936, + "epoch": 0.7038383838383838, + "eps": 0, + "loss/policy_avg": 0.02698252536356449, + "loss/value_avg": 0.2791609764099121, + "lr": 2.2277043274773856e-06, + "objective/entropy": -45.07421112060547, + "objective/kl": 39.21965026855469, + "objective/non_score_reward": -1.9609825611114502, + "objective/rlhf_reward": 14.624954223632812, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009133699350059032, + "policy/clipfrac_avg": 0.004030972719192505, + "policy/entropy_avg": 0.18420028686523438, + "step": 871, + "val/clipfrac_avg": 0.03648557886481285, + "val/num_eos_tokens": 232, + "val/ratio": 0.9881106615066528, + "val/ratio_var": 5.0894170271931216e-05 + }, + { + "episode": 13952, + "epoch": 0.7046464646464646, + "eps": 0, + "loss/policy_avg": -0.12599462270736694, + "loss/value_avg": 0.5007761716842651, + "lr": 2.216598298357646e-06, + "objective/entropy": -52.43424987792969, + "objective/kl": 37.084686279296875, + "objective/non_score_reward": -1.8542344570159912, + "objective/rlhf_reward": 14.18482780456543, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008421384263783693, + "policy/clipfrac_avg": 0.002664695493876934, + "policy/entropy_avg": 0.1716923713684082, + "step": 872, + "val/clipfrac_avg": 0.0010017228778451681, + "val/num_eos_tokens": 274, + "val/ratio": 0.9881088137626648, + "val/ratio_var": 6.610793934669346e-05 + }, + { + "episode": 13968, + "epoch": 0.7054545454545454, + "eps": 0, + "loss/policy_avg": 0.35178622603416443, + "loss/value_avg": 0.5846590995788574, + "lr": 2.2055121329646416e-06, + "objective/entropy": -138.92434692382812, + "objective/kl": 41.75783157348633, + "objective/non_score_reward": -2.0878915786743164, + "objective/rlhf_reward": 13.091796875, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008433774346485734, + "policy/clipfrac_avg": 0.006052501033991575, + "policy/entropy_avg": 0.16450119018554688, + "step": 873, + "val/clipfrac_avg": 0.0322103314101696, + "val/num_eos_tokens": 1144, + "val/ratio": 0.9888837337493896, + "val/ratio_var": 5.522933497559279e-05 + }, + { + "episode": 13984, + "epoch": 0.7062626262626263, + "eps": 0, + "loss/policy_avg": -0.004525661468505859, + "loss/value_avg": 0.5259770750999451, + "lr": 2.1944459104147188e-06, + "objective/entropy": -122.14696502685547, + "objective/kl": 37.81830978393555, + "objective/non_score_reward": -1.8909155130386353, + "objective/rlhf_reward": 13.296585083007812, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0006640740903094411, + "policy/clipfrac_avg": 0.004708480089902878, + "policy/entropy_avg": 0.14041614532470703, + "step": 874, + "val/clipfrac_avg": 0.10728074610233307, + "val/num_eos_tokens": 35, + "val/ratio": 0.990477979183197, + "val/ratio_var": 6.866270996397361e-05 + }, + { + "episode": 14000, + "epoch": 0.7070707070707071, + "eps": 0, + "loss/policy_avg": 0.11862924695014954, + "loss/value_avg": 1.0722770690917969, + "lr": 2.1833997096818897e-06, + "objective/entropy": -54.207332611083984, + "objective/kl": 37.62938690185547, + "objective/non_score_reward": -1.8814692497253418, + "objective/rlhf_reward": 14.2200927734375, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000859625346492976, + "policy/clipfrac_avg": 0.005390159785747528, + "policy/entropy_avg": 0.1755199432373047, + "step": 875, + "val/clipfrac_avg": 0.0593750923871994, + "val/num_eos_tokens": 201, + "val/ratio": 0.9878208637237549, + "val/ratio_var": 6.484645564341918e-05 + }, + { + "episode": 14016, + "epoch": 0.7078787878787879, + "eps": 0, + "loss/policy_avg": -0.2030085027217865, + "loss/value_avg": 0.5594953298568726, + "lr": 2.1723736095972946e-06, + "objective/entropy": -16.795242309570312, + "objective/kl": 44.697776794433594, + "objective/non_score_reward": -2.234889030456543, + "objective/rlhf_reward": 13.140110969543457, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0011914006900042295, + "policy/clipfrac_avg": 0.005575358867645264, + "policy/entropy_avg": 0.2543792724609375, + "step": 876, + "val/clipfrac_avg": 0.02393021248281002, + "val/num_eos_tokens": 24, + "val/ratio": 0.9835648536682129, + "val/ratio_var": 0.00010361165914218873 + }, + { + "episode": 14032, + "epoch": 0.7086868686868687, + "eps": 0, + "loss/policy_avg": 0.05069049820303917, + "loss/value_avg": 0.6454532146453857, + "lr": 2.161367688848618e-06, + "objective/entropy": -105.85357666015625, + "objective/kl": 50.78942108154297, + "objective/non_score_reward": -2.53947114944458, + "objective/rlhf_reward": 13.077716827392578, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008386933477595448, + "policy/clipfrac_avg": 0.003330180887132883, + "policy/entropy_avg": 0.1841726303100586, + "step": 877, + "val/clipfrac_avg": 0.010819870047271252, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886084198951721, + "val/ratio_var": 5.4145024478202686e-05 + }, + { + "episode": 14048, + "epoch": 0.7094949494949495, + "eps": 0, + "loss/policy_avg": -0.32361558079719543, + "loss/value_avg": 0.4275820851325989, + "lr": 2.1503820259795357e-06, + "objective/entropy": -193.71432495117188, + "objective/kl": 41.63684844970703, + "objective/non_score_reward": -2.0818424224853516, + "objective/rlhf_reward": 14.261907577514648, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007008577231317759, + "policy/clipfrac_avg": 0.004103646147996187, + "policy/entropy_avg": 0.1445927619934082, + "step": 878, + "val/clipfrac_avg": 7.62939453125e-06, + "val/num_eos_tokens": 323, + "val/ratio": 0.9903497695922852, + "val/ratio_var": 3.22636442433577e-05 + }, + { + "episode": 14064, + "epoch": 0.7103030303030303, + "eps": 0, + "loss/policy_avg": -0.1645442247390747, + "loss/value_avg": 0.7777354717254639, + "lr": 2.139416699389153e-06, + "objective/entropy": -45.89549255371094, + "objective/kl": 34.282958984375, + "objective/non_score_reward": -1.7141478061676025, + "objective/rlhf_reward": 14.692102432250977, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007650890620425344, + "policy/clipfrac_avg": 0.00561137031763792, + "policy/entropy_avg": 0.15566253662109375, + "step": 879, + "val/clipfrac_avg": 0.10208053886890411, + "val/num_eos_tokens": 49, + "val/ratio": 0.9895084500312805, + "val/ratio_var": 3.380152702447958e-05 + }, + { + "episode": 14080, + "epoch": 0.7111111111111111, + "eps": 0, + "loss/policy_avg": -0.023053357377648354, + "loss/value_avg": 0.521371603012085, + "lr": 2.128471787331446e-06, + "objective/entropy": -50.96327209472656, + "objective/kl": 41.30862808227539, + "objective/non_score_reward": -2.0654313564300537, + "objective/rlhf_reward": 14.450193405151367, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009377336828038096, + "policy/clipfrac_avg": 0.00421791011467576, + "policy/entropy_avg": 0.17507076263427734, + "step": 880, + "val/clipfrac_avg": 0.04448426887392998, + "val/num_eos_tokens": 368, + "val/ratio": 0.9889349937438965, + "val/ratio_var": 3.734078927664086e-05 + }, + { + "episode": 14096, + "epoch": 0.711919191919192, + "eps": 0, + "loss/policy_avg": -0.07007278501987457, + "loss/value_avg": 0.2162141352891922, + "lr": 2.117547367914705e-06, + "objective/entropy": -69.7264175415039, + "objective/kl": 41.05116653442383, + "objective/non_score_reward": -2.052558422088623, + "objective/rlhf_reward": 14.830253601074219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.001040782080963254, + "policy/clipfrac_avg": 0.0033751465380191803, + "policy/entropy_avg": 0.20317459106445312, + "step": 881, + "val/clipfrac_avg": 0.03193364292383194, + "val/num_eos_tokens": 551, + "val/ratio": 0.9861003160476685, + "val/ratio_var": 8.361194340977818e-05 + }, + { + "episode": 14112, + "epoch": 0.7127272727272728, + "eps": 0, + "loss/policy_avg": -0.07847104221582413, + "loss/value_avg": 0.2657424807548523, + "lr": 2.1066435191009717e-06, + "objective/entropy": -86.86426544189453, + "objective/kl": 45.00962829589844, + "objective/non_score_reward": -2.250481367111206, + "objective/rlhf_reward": 13.913580894470215, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010606081923469901, + "policy/clipfrac_avg": 0.0069802612997591496, + "policy/entropy_avg": 0.20689773559570312, + "step": 882, + "val/clipfrac_avg": 0.038475796580314636, + "val/num_eos_tokens": 546, + "val/ratio": 0.985522449016571, + "val/ratio_var": 8.580441499361768e-05 + }, + { + "episode": 14128, + "epoch": 0.7135353535353536, + "eps": 0, + "loss/policy_avg": -0.1323227733373642, + "loss/value_avg": 0.21754738688468933, + "lr": 2.095760318705487e-06, + "objective/entropy": -109.19290161132812, + "objective/kl": 42.61073303222656, + "objective/non_score_reward": -2.1305365562438965, + "objective/rlhf_reward": 14.447587966918945, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010359378065913916, + "policy/clipfrac_avg": 0.006892787292599678, + "policy/entropy_avg": 0.20017433166503906, + "step": 883, + "val/clipfrac_avg": 0.010656789876520634, + "val/num_eos_tokens": 36, + "val/ratio": 0.9864689111709595, + "val/ratio_var": 4.844724026042968e-05 + }, + { + "episode": 14144, + "epoch": 0.7143434343434344, + "eps": 0, + "loss/policy_avg": -0.1507091522216797, + "loss/value_avg": 0.2665415108203888, + "lr": 2.0848978443961314e-06, + "objective/entropy": -119.52471923828125, + "objective/kl": 43.517696380615234, + "objective/non_score_reward": -2.17588472366333, + "objective/rlhf_reward": 14.300678253173828, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000938809709623456, + "policy/clipfrac_avg": 0.012131594121456146, + "policy/entropy_avg": 0.18040943145751953, + "step": 884, + "val/clipfrac_avg": 0.006075765937566757, + "val/num_eos_tokens": 620, + "val/ratio": 0.9873965978622437, + "val/ratio_var": 6.621635111514479e-05 + }, + { + "episode": 14160, + "epoch": 0.7151515151515152, + "eps": 0, + "loss/policy_avg": -0.028530046343803406, + "loss/value_avg": 0.434810072183609, + "lr": 2.074056173692881e-06, + "objective/entropy": -62.273380279541016, + "objective/kl": 45.77116394042969, + "objective/non_score_reward": -2.2885584831237793, + "objective/rlhf_reward": 14.617691040039062, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009718666551634669, + "policy/clipfrac_avg": 0.004924894310534, + "policy/entropy_avg": 0.20779800415039062, + "step": 885, + "val/clipfrac_avg": 0.036309417337179184, + "val/num_eos_tokens": 179, + "val/ratio": 0.9870821237564087, + "val/ratio_var": 8.347785478690639e-05 + }, + { + "episode": 14176, + "epoch": 0.7159595959595959, + "eps": 0, + "loss/policy_avg": -0.15033547580242157, + "loss/value_avg": 0.6043248772621155, + "lr": 2.0632353839672415e-06, + "objective/entropy": -84.31056213378906, + "objective/kl": 32.41828918457031, + "objective/non_score_reward": -1.6209145784378052, + "objective/rlhf_reward": 15.332210540771484, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008589382632635534, + "policy/clipfrac_avg": 0.002752305706962943, + "policy/entropy_avg": 0.18497180938720703, + "step": 886, + "val/clipfrac_avg": 0.03947104513645172, + "val/num_eos_tokens": 180, + "val/ratio": 0.9879313707351685, + "val/ratio_var": 5.601351585937664e-05 + }, + { + "episode": 14192, + "epoch": 0.7167676767676767, + "eps": 0, + "loss/policy_avg": 0.2279771864414215, + "loss/value_avg": 1.2074341773986816, + "lr": 2.0524355524417017e-06, + "objective/entropy": -51.082984924316406, + "objective/kl": 36.84368133544922, + "objective/non_score_reward": -1.842184066772461, + "objective/rlhf_reward": 14.087503433227539, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010461971396580338, + "policy/clipfrac_avg": 0.004587469156831503, + "policy/entropy_avg": 0.22597980499267578, + "step": 887, + "val/clipfrac_avg": 0.015519710257649422, + "val/num_eos_tokens": 448, + "val/ratio": 0.9849185943603516, + "val/ratio_var": 6.34926327620633e-05 + }, + { + "episode": 14208, + "epoch": 0.7175757575757575, + "eps": 0, + "loss/policy_avg": -0.026267291978001595, + "loss/value_avg": 0.5965927839279175, + "lr": 2.041656756189184e-06, + "objective/entropy": -154.47012329101562, + "objective/kl": 42.720699310302734, + "objective/non_score_reward": -2.1360349655151367, + "objective/rlhf_reward": 14.957715034484863, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009299394441768527, + "policy/clipfrac_avg": 0.008321288973093033, + "policy/entropy_avg": 0.1751893162727356, + "step": 888, + "val/clipfrac_avg": 0.01875000074505806, + "val/num_eos_tokens": 428, + "val/ratio": 0.9883604049682617, + "val/ratio_var": 7.325677870539948e-05 + }, + { + "episode": 14224, + "epoch": 0.7183838383838383, + "eps": 0, + "loss/policy_avg": 0.40078702569007874, + "loss/value_avg": 1.1534854173660278, + "lr": 2.030899072132493e-06, + "objective/entropy": -128.48831176757812, + "objective/kl": 33.898155212402344, + "objective/non_score_reward": -1.6949079036712646, + "objective/rlhf_reward": 12.598060607910156, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006705562118440866, + "policy/clipfrac_avg": 0.005831941030919552, + "policy/entropy_avg": 0.21662330627441406, + "step": 889, + "val/clipfrac_avg": 0.02483724057674408, + "val/num_eos_tokens": 47, + "val/ratio": 0.9912885427474976, + "val/ratio_var": 7.058407936710864e-05 + }, + { + "episode": 14240, + "epoch": 0.7191919191919192, + "eps": 0, + "loss/policy_avg": -0.0855930745601654, + "loss/value_avg": 0.32921552658081055, + "lr": 2.0201625770437647e-06, + "objective/entropy": -86.9806137084961, + "objective/kl": 40.06652069091797, + "objective/non_score_reward": -2.003325939178467, + "objective/rlhf_reward": 14.035737037658691, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008815613109618425, + "policy/clipfrac_avg": 0.006017718929797411, + "policy/entropy_avg": 0.1755352020263672, + "step": 890, + "val/clipfrac_avg": 0.06256753951311111, + "val/num_eos_tokens": 20, + "val/ratio": 0.9872984290122986, + "val/ratio_var": 8.527067984687164e-05 + }, + { + "episode": 14256, + "epoch": 0.72, + "eps": 0, + "loss/policy_avg": -0.17781905829906464, + "loss/value_avg": 0.2275725156068802, + "lr": 2.00944734754392e-06, + "objective/entropy": -99.16207122802734, + "objective/kl": 41.95650863647461, + "objective/non_score_reward": -2.097825288772583, + "objective/rlhf_reward": 13.74592399597168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007897773757576942, + "policy/clipfrac_avg": 0.0043031517416238785, + "policy/entropy_avg": 0.1585559844970703, + "step": 891, + "val/clipfrac_avg": 0.012334126979112625, + "val/num_eos_tokens": 424, + "val/ratio": 0.9894897937774658, + "val/ratio_var": 5.430886085378006e-05 + }, + { + "episode": 14272, + "epoch": 0.7208080808080808, + "eps": 0, + "loss/policy_avg": -0.36396488547325134, + "loss/value_avg": 0.8662234544754028, + "lr": 1.9987534601021225e-06, + "objective/entropy": -61.42802047729492, + "objective/kl": 37.07469177246094, + "objective/non_score_reward": -1.8537347316741943, + "objective/rlhf_reward": 13.044702529907227, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008486385922878981, + "policy/clipfrac_avg": 0.006757801398634911, + "policy/entropy_avg": 0.1866321563720703, + "step": 892, + "val/clipfrac_avg": 0.054801926016807556, + "val/num_eos_tokens": 584, + "val/ratio": 0.9883260726928711, + "val/ratio_var": 6.7303488322068e-05 + }, + { + "episode": 14288, + "epoch": 0.7216161616161616, + "eps": 0, + "loss/policy_avg": -0.12171509861946106, + "loss/value_avg": 0.3499939441680908, + "lr": 1.988080991035223e-06, + "objective/entropy": -58.38874053955078, + "objective/kl": 42.03372573852539, + "objective/non_score_reward": -2.1016862392425537, + "objective/rlhf_reward": 12.828001022338867, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008078413084149361, + "policy/clipfrac_avg": 0.004710954613983631, + "policy/entropy_avg": 0.16219139099121094, + "step": 893, + "val/clipfrac_avg": 0.03385913744568825, + "val/num_eos_tokens": 211, + "val/ratio": 0.9909918904304504, + "val/ratio_var": 4.974031253368594e-05 + }, + { + "episode": 14304, + "epoch": 0.7224242424242424, + "eps": 0, + "loss/policy_avg": 0.15615974366664886, + "loss/value_avg": 0.4985978901386261, + "lr": 1.977430016507222e-06, + "objective/entropy": -72.03929138183594, + "objective/kl": 43.86260986328125, + "objective/non_score_reward": -2.1931304931640625, + "objective/rlhf_reward": 13.322494506835938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009859772399067879, + "policy/clipfrac_avg": 0.004861406981945038, + "policy/entropy_avg": 0.19806480407714844, + "step": 894, + "val/clipfrac_avg": 0.008869262412190437, + "val/num_eos_tokens": 389, + "val/ratio": 0.9869673252105713, + "val/ratio_var": 5.9525649703573436e-05 + }, + { + "episode": 14320, + "epoch": 0.7232323232323232, + "eps": 0, + "loss/policy_avg": -0.018653089180588722, + "loss/value_avg": 0.3236011266708374, + "lr": 1.966800612528723e-06, + "objective/entropy": -30.8804874420166, + "objective/kl": 38.795677185058594, + "objective/non_score_reward": -1.9397838115692139, + "objective/rlhf_reward": 12.958653450012207, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011067797895520926, + "policy/clipfrac_avg": 0.004753993824124336, + "policy/entropy_avg": 0.21401214599609375, + "step": 895, + "val/clipfrac_avg": 0.013285137712955475, + "val/num_eos_tokens": 190, + "val/ratio": 0.9844212532043457, + "val/ratio_var": 5.578770287684165e-05 + }, + { + "episode": 14336, + "epoch": 0.724040404040404, + "eps": 0, + "loss/policy_avg": -0.04005177319049835, + "loss/value_avg": 0.564488410949707, + "lr": 1.956192854956397e-06, + "objective/entropy": -88.01570129394531, + "objective/kl": 40.9593505859375, + "objective/non_score_reward": -2.0479674339294434, + "objective/rlhf_reward": 13.350469589233398, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010309319477528334, + "policy/clipfrac_avg": 0.005483411252498627, + "policy/entropy_avg": 0.21037864685058594, + "step": 896, + "val/clipfrac_avg": 0.01046415138989687, + "val/num_eos_tokens": 539, + "val/ratio": 0.9865807890892029, + "val/ratio_var": 9.776368096936494e-05 + }, + { + "episode": 14352, + "epoch": 0.7248484848484849, + "eps": 0, + "loss/policy_avg": -0.1358407437801361, + "loss/value_avg": 0.536889910697937, + "lr": 1.945606819492429e-06, + "objective/entropy": -98.2094955444336, + "objective/kl": 37.47423553466797, + "objective/non_score_reward": -1.8737118244171143, + "objective/rlhf_reward": 13.634100914001465, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007985340780578554, + "policy/clipfrac_avg": 0.005063176155090332, + "policy/entropy_avg": 0.14582359790802002, + "step": 897, + "val/clipfrac_avg": 0.06062069162726402, + "val/num_eos_tokens": 1182, + "val/ratio": 0.9903557300567627, + "val/ratio_var": 5.06751166540198e-05 + }, + { + "episode": 14368, + "epoch": 0.7256565656565657, + "eps": 0, + "loss/policy_avg": 0.011427253484725952, + "loss/value_avg": 0.263162761926651, + "lr": 1.9350425816839875e-06, + "objective/entropy": -110.30941772460938, + "objective/kl": 41.13695526123047, + "objective/non_score_reward": -2.0568478107452393, + "objective/rlhf_reward": 13.72440242767334, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011626393534243107, + "policy/clipfrac_avg": 0.006754214409738779, + "policy/entropy_avg": 0.2226581573486328, + "step": 898, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 10, + "val/ratio": 0.9850178956985474, + "val/ratio_var": 8.876781794242561e-05 + }, + { + "episode": 14384, + "epoch": 0.7264646464646465, + "eps": 0, + "loss/policy_avg": 0.11541193723678589, + "loss/value_avg": 0.4631091356277466, + "lr": 1.9245002169226814e-06, + "objective/entropy": -101.70489501953125, + "objective/kl": 48.557960510253906, + "objective/non_score_reward": -2.4278979301452637, + "objective/rlhf_reward": 13.314289093017578, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009826980531215668, + "policy/clipfrac_avg": 0.004610124975442886, + "policy/entropy_avg": 0.1724681854248047, + "step": 899, + "val/clipfrac_avg": 0.013620924204587936, + "val/num_eos_tokens": 516, + "val/ratio": 0.9876973628997803, + "val/ratio_var": 5.4934793297434226e-05 + }, + { + "episode": 14400, + "epoch": 0.7272727272727273, + "eps": 0, + "loss/policy_avg": 0.07936800271272659, + "loss/value_avg": 0.6616516709327698, + "lr": 1.913979800444021e-06, + "objective/entropy": -91.96019744873047, + "objective/kl": 41.26939010620117, + "objective/non_score_reward": -2.063469409942627, + "objective/rlhf_reward": 12.522467613220215, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0009332017507404089, + "policy/clipfrac_avg": 0.0043076397851109505, + "policy/entropy_avg": 0.18123245239257812, + "step": 900, + "val/clipfrac_avg": 0.01476163137704134, + "val/num_eos_tokens": 595, + "val/ratio": 0.9875973463058472, + "val/ratio_var": 7.475823076674715e-05 + }, + { + "episode": 14416, + "epoch": 0.7280808080808081, + "eps": 0, + "loss/policy_avg": -0.02671997621655464, + "loss/value_avg": 0.7174835205078125, + "lr": 1.9034814073268898e-06, + "objective/entropy": -59.236602783203125, + "objective/kl": 39.73054122924805, + "objective/non_score_reward": -1.9865270853042603, + "objective/rlhf_reward": 12.435348510742188, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011704354546964169, + "policy/clipfrac_avg": 0.004937891382724047, + "policy/entropy_avg": 0.22887516021728516, + "step": 901, + "val/clipfrac_avg": 0.13261640071868896, + "val/num_eos_tokens": 121, + "val/ratio": 0.9848372936248779, + "val/ratio_var": 8.883672853698954e-05 + }, + { + "episode": 14432, + "epoch": 0.7288888888888889, + "eps": 0, + "loss/policy_avg": 0.05788304656744003, + "loss/value_avg": 0.2591496407985687, + "lr": 1.8930051124929904e-06, + "objective/entropy": -53.46650695800781, + "objective/kl": 46.268035888671875, + "objective/non_score_reward": -2.313401937484741, + "objective/rlhf_reward": 14.38972282409668, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010022292844951153, + "policy/clipfrac_avg": 0.004066841676831245, + "policy/entropy_avg": 0.20316123962402344, + "step": 902, + "val/clipfrac_avg": 0.013965151272714138, + "val/num_eos_tokens": 455, + "val/ratio": 0.9866280555725098, + "val/ratio_var": 5.229823727859184e-05 + }, + { + "episode": 14448, + "epoch": 0.7296969696969697, + "eps": 0, + "loss/policy_avg": -0.11720985174179077, + "loss/value_avg": 0.23323997855186462, + "lr": 1.8825509907063328e-06, + "objective/entropy": -65.6883773803711, + "objective/kl": 46.209991455078125, + "objective/non_score_reward": -2.310499668121338, + "objective/rlhf_reward": 16.78325080871582, + "objective/scores": 19.125, + "policy/approxkl_avg": 0.0010066109243780375, + "policy/clipfrac_avg": 0.005406823009252548, + "policy/entropy_avg": 0.20101547241210938, + "step": 903, + "val/clipfrac_avg": 0.015051587484776974, + "val/num_eos_tokens": 205, + "val/ratio": 0.9865299463272095, + "val/ratio_var": 5.01565846207086e-05 + }, + { + "episode": 14464, + "epoch": 0.7305050505050505, + "eps": 0, + "loss/policy_avg": -0.4681462049484253, + "loss/value_avg": 0.8052223920822144, + "lr": 1.87211911657268e-06, + "objective/entropy": -118.0733642578125, + "objective/kl": 37.29925537109375, + "objective/non_score_reward": -1.8649628162384033, + "objective/rlhf_reward": 15.885037422180176, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0007588897133246064, + "policy/clipfrac_avg": 0.0023906652349978685, + "policy/entropy_avg": 0.15973949432373047, + "step": 904, + "val/clipfrac_avg": 0.13810205459594727, + "val/num_eos_tokens": 155, + "val/ratio": 0.9902135729789734, + "val/ratio_var": 7.25651771062985e-05 + }, + { + "episode": 14480, + "epoch": 0.7313131313131314, + "eps": 0, + "loss/policy_avg": 0.19160020351409912, + "loss/value_avg": 0.9071890115737915, + "lr": 1.8617095645390332e-06, + "objective/entropy": -22.496440887451172, + "objective/kl": 39.160484313964844, + "objective/non_score_reward": -1.958024263381958, + "objective/rlhf_reward": 12.323225975036621, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0010718535631895065, + "policy/clipfrac_avg": 0.004276500083506107, + "policy/entropy_avg": 0.19223785400390625, + "step": 905, + "val/clipfrac_avg": 0.03053325042128563, + "val/num_eos_tokens": 240, + "val/ratio": 0.9871090650558472, + "val/ratio_var": 4.5721961214439943e-05 + }, + { + "episode": 14496, + "epoch": 0.7321212121212122, + "eps": 0, + "loss/policy_avg": -0.11877916008234024, + "loss/value_avg": 0.6873422265052795, + "lr": 1.8513224088930814e-06, + "objective/entropy": -52.41043472290039, + "objective/kl": 30.631162643432617, + "objective/non_score_reward": -1.5315582752227783, + "objective/rlhf_reward": 16.296566009521484, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.001107763615436852, + "policy/clipfrac_avg": 0.006524171680212021, + "policy/entropy_avg": 0.2064342498779297, + "step": 906, + "val/clipfrac_avg": 0.000294811325147748, + "val/num_eos_tokens": 261, + "val/ratio": 0.9861389398574829, + "val/ratio_var": 6.195842433953658e-05 + }, + { + "episode": 14512, + "epoch": 0.7329292929292929, + "eps": 0, + "loss/policy_avg": 0.10565789043903351, + "loss/value_avg": 0.5067005157470703, + "lr": 1.8409577237626935e-06, + "objective/entropy": -61.856544494628906, + "objective/kl": 43.33234405517578, + "objective/non_score_reward": -2.1666173934936523, + "objective/rlhf_reward": 13.145882606506348, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010527942795306444, + "policy/clipfrac_avg": 0.004575061611831188, + "policy/entropy_avg": 0.18899154663085938, + "step": 907, + "val/clipfrac_avg": 0.005580357275903225, + "val/num_eos_tokens": 496, + "val/ratio": 0.9872788190841675, + "val/ratio_var": 7.066375110298395e-05 + }, + { + "episode": 14528, + "epoch": 0.7337373737373737, + "eps": 0, + "loss/policy_avg": 0.04191252216696739, + "loss/value_avg": 0.30707108974456787, + "lr": 1.8306155831153699e-06, + "objective/entropy": -91.67915344238281, + "objective/kl": 37.619895935058594, + "objective/non_score_reward": -1.8809947967529297, + "objective/rlhf_reward": 14.22056770324707, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009494669502601027, + "policy/clipfrac_avg": 0.007099117152392864, + "policy/entropy_avg": 0.17319107055664062, + "step": 908, + "val/clipfrac_avg": 0.07242873311042786, + "val/num_eos_tokens": 437, + "val/ratio": 0.9883796572685242, + "val/ratio_var": 7.36714864615351e-05 + }, + { + "episode": 14544, + "epoch": 0.7345454545454545, + "eps": 0, + "loss/policy_avg": 0.15317559242248535, + "loss/value_avg": 0.7379442453384399, + "lr": 1.8202960607577246e-06, + "objective/entropy": -92.03844451904297, + "objective/kl": 36.91444396972656, + "objective/non_score_reward": -1.8457221984863281, + "objective/rlhf_reward": 13.802715301513672, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0009770962642505765, + "policy/clipfrac_avg": 0.005569060333073139, + "policy/entropy_avg": 0.18958091735839844, + "step": 909, + "val/clipfrac_avg": 0.04051949828863144, + "val/num_eos_tokens": 56, + "val/ratio": 0.988923192024231, + "val/ratio_var": 6.142368511063978e-05 + }, + { + "episode": 14560, + "epoch": 0.7353535353535353, + "eps": 0, + "loss/policy_avg": 0.1602431833744049, + "loss/value_avg": 0.3025469183921814, + "lr": 1.809999230334958e-06, + "objective/entropy": -111.80148315429688, + "objective/kl": 38.83802032470703, + "objective/non_score_reward": -1.9419012069702148, + "objective/rlhf_reward": 14.401848793029785, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000904000480659306, + "policy/clipfrac_avg": 0.005316127557307482, + "policy/entropy_avg": 0.17273902893066406, + "step": 910, + "val/clipfrac_avg": 0.00018382353300694376, + "val/num_eos_tokens": 342, + "val/ratio": 0.9891130924224854, + "val/ratio_var": 7.234779332065955e-05 + }, + { + "episode": 14576, + "epoch": 0.7361616161616161, + "eps": 0, + "loss/policy_avg": 0.09882547706365585, + "loss/value_avg": 0.6121327877044678, + "lr": 1.7997251653303249e-06, + "objective/entropy": -98.02986145019531, + "objective/kl": 44.55670166015625, + "objective/non_score_reward": -2.227834939956665, + "objective/rlhf_reward": 13.092477798461914, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009336632210761309, + "policy/clipfrac_avg": 0.004822542425245047, + "policy/entropy_avg": 0.17920780181884766, + "step": 911, + "val/clipfrac_avg": 0.0050772735849022865, + "val/num_eos_tokens": 426, + "val/ratio": 0.9875223636627197, + "val/ratio_var": 6.380004924722016e-05 + }, + { + "episode": 14592, + "epoch": 0.7369696969696969, + "eps": 0, + "loss/policy_avg": -0.15951846539974213, + "loss/value_avg": 0.5648845434188843, + "lr": 1.7894739390646227e-06, + "objective/entropy": -57.16059875488281, + "objective/kl": 36.05466079711914, + "objective/non_score_reward": -1.802733063697815, + "objective/rlhf_reward": 13.119142532348633, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009670511353760958, + "policy/clipfrac_avg": 0.00586615689098835, + "policy/entropy_avg": 0.1721019744873047, + "step": 912, + "val/clipfrac_avg": 0.0008223684271797538, + "val/num_eos_tokens": 45, + "val/ratio": 0.9892112016677856, + "val/ratio_var": 4.9591733841225505e-05 + }, + { + "episode": 14608, + "epoch": 0.7377777777777778, + "eps": 0, + "loss/policy_avg": 0.1940194070339203, + "loss/value_avg": 0.3815060555934906, + "lr": 1.7792456246956535e-06, + "objective/entropy": -146.6658935546875, + "objective/kl": 46.18011474609375, + "objective/non_score_reward": -2.3090057373046875, + "objective/rlhf_reward": 14.511306762695312, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009307245491072536, + "policy/clipfrac_avg": 0.006143895909190178, + "policy/entropy_avg": 0.18556424975395203, + "step": 913, + "val/clipfrac_avg": 0.0180903859436512, + "val/num_eos_tokens": 698, + "val/ratio": 0.9874189496040344, + "val/ratio_var": 0.00011522904242156073 + }, + { + "episode": 14624, + "epoch": 0.7385858585858586, + "eps": 0, + "loss/policy_avg": -0.02095976285636425, + "loss/value_avg": 0.7224372625350952, + "lr": 1.7690402952177117e-06, + "objective/entropy": -84.84890747070312, + "objective/kl": 35.78252410888672, + "objective/non_score_reward": -1.7891263961791992, + "objective/rlhf_reward": 13.2655611038208, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009321504039689898, + "policy/clipfrac_avg": 0.005007730796933174, + "policy/entropy_avg": 0.18702125549316406, + "step": 914, + "val/clipfrac_avg": 0.023768942803144455, + "val/num_eos_tokens": 403, + "val/ratio": 0.9874284267425537, + "val/ratio_var": 8.100390550680459e-05 + }, + { + "episode": 14640, + "epoch": 0.7393939393939394, + "eps": 0, + "loss/policy_avg": -0.3134731650352478, + "loss/value_avg": 0.5765621066093445, + "lr": 1.7588580234610592e-06, + "objective/entropy": -63.22113037109375, + "objective/kl": 38.5015869140625, + "objective/non_score_reward": -1.925079345703125, + "objective/rlhf_reward": 13.918670654296875, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008501409320160747, + "policy/clipfrac_avg": 0.004706401377916336, + "policy/entropy_avg": 0.1600513458251953, + "step": 915, + "val/clipfrac_avg": 0.06931675970554352, + "val/num_eos_tokens": 171, + "val/ratio": 0.9882992506027222, + "val/ratio_var": 6.202542135724798e-05 + }, + { + "episode": 14656, + "epoch": 0.7402020202020202, + "eps": 0, + "loss/policy_avg": -0.2850199043750763, + "loss/value_avg": 0.8679437637329102, + "lr": 1.7486988820914063e-06, + "objective/entropy": -43.271400451660156, + "objective/kl": 42.955291748046875, + "objective/non_score_reward": -2.1477646827697754, + "objective/rlhf_reward": 13.875673294067383, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009495788253843784, + "policy/clipfrac_avg": 0.005583056248724461, + "policy/entropy_avg": 0.17831039428710938, + "step": 916, + "val/clipfrac_avg": 0.06379853188991547, + "val/num_eos_tokens": 103, + "val/ratio": 0.9876798391342163, + "val/ratio_var": 4.0028760849963874e-05 + }, + { + "episode": 14672, + "epoch": 0.741010101010101, + "eps": 0, + "loss/policy_avg": 0.21622401475906372, + "loss/value_avg": 0.22584834694862366, + "lr": 1.7385629436093958e-06, + "objective/entropy": -79.67193603515625, + "objective/kl": 44.639923095703125, + "objective/non_score_reward": -2.2319962978363037, + "objective/rlhf_reward": 13.463315963745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008941494161263108, + "policy/clipfrac_avg": 0.006674608215689659, + "policy/entropy_avg": 0.1719951629638672, + "step": 917, + "val/clipfrac_avg": 0.029584797099232674, + "val/num_eos_tokens": 50, + "val/ratio": 0.9886019825935364, + "val/ratio_var": 4.6610963181592524e-05 + }, + { + "episode": 14688, + "epoch": 0.7418181818181818, + "eps": 0, + "loss/policy_avg": -0.0006486065685749054, + "loss/value_avg": 0.2962718605995178, + "lr": 1.728450280350079e-06, + "objective/entropy": -68.00552368164062, + "objective/kl": 38.6593132019043, + "objective/non_score_reward": -1.9329655170440674, + "objective/rlhf_reward": 13.887346267700195, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008251155959442258, + "policy/clipfrac_avg": 0.005509347654879093, + "policy/entropy_avg": 0.1643085479736328, + "step": 918, + "val/clipfrac_avg": 0.003826581174507737, + "val/num_eos_tokens": 44, + "val/ratio": 0.9888970851898193, + "val/ratio_var": 3.0508910640492104e-05 + }, + { + "episode": 14704, + "epoch": 0.7426262626262626, + "eps": 0, + "loss/policy_avg": -0.3462671935558319, + "loss/value_avg": 0.7152901887893677, + "lr": 1.7183609644824096e-06, + "objective/entropy": -93.26936340332031, + "objective/kl": 40.42371368408203, + "objective/non_score_reward": -2.021185874938965, + "objective/rlhf_reward": 15.166314125061035, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009743100963532925, + "policy/clipfrac_avg": 0.0028937258757650852, + "policy/entropy_avg": 0.18361568450927734, + "step": 919, + "val/clipfrac_avg": 0.049027521163225174, + "val/num_eos_tokens": 136, + "val/ratio": 0.9877507090568542, + "val/ratio_var": 5.374600732466206e-05 + }, + { + "episode": 14720, + "epoch": 0.7434343434343434, + "eps": 0, + "loss/policy_avg": 0.1274300217628479, + "loss/value_avg": 1.230905294418335, + "lr": 1.708295068008718e-06, + "objective/entropy": -81.48149108886719, + "objective/kl": 36.17670440673828, + "objective/non_score_reward": -1.8088352680206299, + "objective/rlhf_reward": 12.589601516723633, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008495738729834557, + "policy/clipfrac_avg": 0.0038070818409323692, + "policy/entropy_avg": 0.16942787170410156, + "step": 920, + "val/clipfrac_avg": 0.007427365053445101, + "val/num_eos_tokens": 270, + "val/ratio": 0.9902092218399048, + "val/ratio_var": 5.2669711294583976e-05 + }, + { + "episode": 14736, + "epoch": 0.7442424242424243, + "eps": 0, + "loss/policy_avg": -0.257866770029068, + "loss/value_avg": 0.5942734479904175, + "lr": 1.6982526627642043e-06, + "objective/entropy": -74.04690551757812, + "objective/kl": 42.419307708740234, + "objective/non_score_reward": -2.1209654808044434, + "objective/rlhf_reward": 13.511846542358398, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000907333567738533, + "policy/clipfrac_avg": 0.004857117310166359, + "policy/entropy_avg": 0.18361282348632812, + "step": 921, + "val/clipfrac_avg": 0.043217435479164124, + "val/num_eos_tokens": 56, + "val/ratio": 0.9877738356590271, + "val/ratio_var": 6.433074304368347e-05 + }, + { + "episode": 14752, + "epoch": 0.7450505050505051, + "eps": 0, + "loss/policy_avg": 0.12189102917909622, + "loss/value_avg": 0.41360408067703247, + "lr": 1.6882338204164212e-06, + "objective/entropy": -54.975791931152344, + "objective/kl": 35.72700881958008, + "objective/non_score_reward": -1.7863504886627197, + "objective/rlhf_reward": 13.39333724975586, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010183433769270778, + "policy/clipfrac_avg": 0.003768398892134428, + "policy/entropy_avg": 0.1844940185546875, + "step": 922, + "val/clipfrac_avg": 0.0008610248914919794, + "val/num_eos_tokens": 362, + "val/ratio": 0.9872738122940063, + "val/ratio_var": 3.0919560231268406e-05 + }, + { + "episode": 14768, + "epoch": 0.7458585858585859, + "eps": 0, + "loss/policy_avg": -0.07059650868177414, + "loss/value_avg": 0.6463966369628906, + "lr": 1.67823861246477e-06, + "objective/entropy": -15.746962547302246, + "objective/kl": 40.59729766845703, + "objective/non_score_reward": -2.029865026473999, + "objective/rlhf_reward": 12.388103485107422, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010770086664706469, + "policy/clipfrac_avg": 0.005413630045950413, + "policy/entropy_avg": 0.21207809448242188, + "step": 923, + "val/clipfrac_avg": 0.00856552179902792, + "val/num_eos_tokens": 175, + "val/ratio": 0.9861268401145935, + "val/ratio_var": 5.63512985536363e-05 + }, + { + "episode": 14784, + "epoch": 0.7466666666666667, + "eps": 0, + "loss/policy_avg": 0.06915351748466492, + "loss/value_avg": 1.4651566743850708, + "lr": 1.6682671102399806e-06, + "objective/entropy": -118.91422271728516, + "objective/kl": 34.25519943237305, + "objective/non_score_reward": -1.7127599716186523, + "objective/rlhf_reward": 12.224739074707031, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0007098158821463585, + "policy/clipfrac_avg": 0.0022262982092797756, + "policy/entropy_avg": 0.1386256217956543, + "step": 924, + "val/clipfrac_avg": 0.06050518900156021, + "val/num_eos_tokens": 270, + "val/ratio": 0.9912421703338623, + "val/ratio_var": 4.530951628112234e-05 + }, + { + "episode": 14800, + "epoch": 0.7474747474747475, + "eps": 0, + "loss/policy_avg": 0.18248242139816284, + "loss/value_avg": 1.6025516986846924, + "lr": 1.6583193849036095e-06, + "objective/entropy": -81.5009765625, + "objective/kl": 38.95854949951172, + "objective/non_score_reward": -1.947927474975586, + "objective/rlhf_reward": 15.450510025024414, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009277502540498972, + "policy/clipfrac_avg": 0.004952313844114542, + "policy/entropy_avg": 0.16534215211868286, + "step": 925, + "val/clipfrac_avg": 0.016582708805799484, + "val/num_eos_tokens": 323, + "val/ratio": 0.9883300065994263, + "val/ratio_var": 5.890153624932282e-05 + }, + { + "episode": 14816, + "epoch": 0.7482828282828283, + "eps": 0, + "loss/policy_avg": -0.16795751452445984, + "loss/value_avg": 0.3992899954319, + "lr": 1.6483955074475278e-06, + "objective/entropy": -46.540870666503906, + "objective/kl": 35.18474578857422, + "objective/non_score_reward": -1.75923752784729, + "objective/rlhf_reward": 13.225137710571289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009727986180223525, + "policy/clipfrac_avg": 0.0023795529268682003, + "policy/entropy_avg": 0.21690034866333008, + "step": 926, + "val/clipfrac_avg": 0.0444885715842247, + "val/num_eos_tokens": 163, + "val/ratio": 0.9869061708450317, + "val/ratio_var": 6.481046148110181e-05 + }, + { + "episode": 14832, + "epoch": 0.7490909090909091, + "eps": 0, + "loss/policy_avg": -0.3186694383621216, + "loss/value_avg": 0.933090090751648, + "lr": 1.6384955486934157e-06, + "objective/entropy": -73.27378845214844, + "objective/kl": 33.047142028808594, + "objective/non_score_reward": -1.6523573398590088, + "objective/rlhf_reward": 13.33201789855957, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008205980993807316, + "policy/clipfrac_avg": 0.0039194016717374325, + "policy/entropy_avg": 0.17404580116271973, + "step": 927, + "val/clipfrac_avg": 0.07463380694389343, + "val/num_eos_tokens": 165, + "val/ratio": 0.9890973567962646, + "val/ratio_var": 8.332987636094913e-05 + }, + { + "episode": 14848, + "epoch": 0.74989898989899, + "eps": 0, + "loss/policy_avg": -0.018651030957698822, + "loss/value_avg": 0.23730480670928955, + "lr": 1.6286195792922655e-06, + "objective/entropy": -108.95191192626953, + "objective/kl": 41.2000846862793, + "objective/non_score_reward": -2.060004472732544, + "objective/rlhf_reward": 14.135308265686035, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007390918326564133, + "policy/clipfrac_avg": 0.0021800659596920013, + "policy/entropy_avg": 0.15365886688232422, + "step": 928, + "val/clipfrac_avg": 0.02866453304886818, + "val/num_eos_tokens": 497, + "val/ratio": 0.989920973777771, + "val/ratio_var": 5.118203625897877e-05 + }, + { + "episode": 14864, + "epoch": 0.7507070707070707, + "eps": 0, + "loss/policy_avg": 0.11356082558631897, + "loss/value_avg": 0.5069226622581482, + "lr": 1.6187676697238564e-06, + "objective/entropy": -53.78815841674805, + "objective/kl": 47.88815689086914, + "objective/non_score_reward": -2.3944079875946045, + "objective/rlhf_reward": 12.574342727661133, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008938192040659487, + "policy/clipfrac_avg": 0.003149396274238825, + "policy/entropy_avg": 0.181840181350708, + "step": 929, + "val/clipfrac_avg": 0.0172991082072258, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889012575149536, + "val/ratio_var": 3.912192551069893e-05 + }, + { + "episode": 14880, + "epoch": 0.7515151515151515, + "eps": 0, + "loss/policy_avg": -0.07865975797176361, + "loss/value_avg": 0.41433531045913696, + "lr": 1.6089398902962767e-06, + "objective/entropy": -104.1807861328125, + "objective/kl": 37.49846649169922, + "objective/non_score_reward": -1.8749234676361084, + "objective/rlhf_reward": 15.953201293945312, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009115774882957339, + "policy/clipfrac_avg": 0.005451949778944254, + "policy/entropy_avg": 0.1745290756225586, + "step": 930, + "val/clipfrac_avg": 0.058040205389261246, + "val/num_eos_tokens": 27, + "val/ratio": 0.9887331128120422, + "val/ratio_var": 6.705939449602738e-05 + }, + { + "episode": 14896, + "epoch": 0.7523232323232323, + "eps": 0, + "loss/policy_avg": -0.035402484238147736, + "loss/value_avg": 0.5358971357345581, + "lr": 1.5991363111454023e-06, + "objective/entropy": -97.50758361816406, + "objective/kl": 42.68102264404297, + "objective/non_score_reward": -2.1340510845184326, + "objective/rlhf_reward": 13.498762130737305, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008075715741142631, + "policy/clipfrac_avg": 0.011311197653412819, + "policy/entropy_avg": 0.1630105972290039, + "step": 931, + "val/clipfrac_avg": 0.003885046113282442, + "val/num_eos_tokens": 302, + "val/ratio": 0.9890309572219849, + "val/ratio_var": 5.2104482165304944e-05 + }, + { + "episode": 14912, + "epoch": 0.7531313131313131, + "eps": 0, + "loss/policy_avg": -0.30045461654663086, + "loss/value_avg": 0.872713565826416, + "lr": 1.5893570022344124e-06, + "objective/entropy": -78.21574401855469, + "objective/kl": 32.467742919921875, + "objective/non_score_reward": -1.623387098312378, + "objective/rlhf_reward": 14.372706413269043, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008647768991068006, + "policy/clipfrac_avg": 0.005656212102621794, + "policy/entropy_avg": 0.17721939086914062, + "step": 932, + "val/clipfrac_avg": 0.0015919646248221397, + "val/num_eos_tokens": 12, + "val/ratio": 0.9885766506195068, + "val/ratio_var": 6.424664752557874e-05 + }, + { + "episode": 14928, + "epoch": 0.7539393939393939, + "eps": 0, + "loss/policy_avg": -0.012097490951418877, + "loss/value_avg": 0.7004050016403198, + "lr": 1.5796020333532696e-06, + "objective/entropy": -61.095802307128906, + "objective/kl": 36.504150390625, + "objective/non_score_reward": -1.8252074718475342, + "objective/rlhf_reward": 12.393542289733887, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008130621863529086, + "policy/clipfrac_avg": 0.005108194425702095, + "policy/entropy_avg": 0.16521072387695312, + "step": 933, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 182, + "val/ratio": 0.9887269735336304, + "val/ratio_var": 6.136602314654738e-05 + }, + { + "episode": 14944, + "epoch": 0.7547474747474747, + "eps": 0, + "loss/policy_avg": 0.03149443119764328, + "loss/value_avg": 0.3308320641517639, + "lr": 1.5698714741182459e-06, + "objective/entropy": -110.07270812988281, + "objective/kl": 34.973388671875, + "objective/non_score_reward": -1.7486693859100342, + "objective/rlhf_reward": 14.548205375671387, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00091616774443537, + "policy/clipfrac_avg": 0.006772395223379135, + "policy/entropy_avg": 0.17972993850708008, + "step": 934, + "val/clipfrac_avg": 0.0015980113530531526, + "val/num_eos_tokens": 350, + "val/ratio": 0.9878275394439697, + "val/ratio_var": 8.512718341080472e-05 + }, + { + "episode": 14960, + "epoch": 0.7555555555555555, + "eps": 0, + "loss/policy_avg": -0.03353989124298096, + "loss/value_avg": 0.4103987216949463, + "lr": 1.5601653939714073e-06, + "objective/entropy": -25.034250259399414, + "objective/kl": 39.53004837036133, + "objective/non_score_reward": -1.976502537727356, + "objective/rlhf_reward": 14.281310081481934, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102006527595222, + "policy/clipfrac_avg": 0.00497299712151289, + "policy/entropy_avg": 0.19387245178222656, + "step": 935, + "val/clipfrac_avg": 0.0242628026753664, + "val/num_eos_tokens": 73, + "val/ratio": 0.9875220060348511, + "val/ratio_var": 6.656184268649668e-05 + }, + { + "episode": 14976, + "epoch": 0.7563636363636363, + "eps": 0, + "loss/policy_avg": -0.1474141776561737, + "loss/value_avg": 0.527273416519165, + "lr": 1.5504838621801272e-06, + "objective/entropy": -60.77607345581055, + "objective/kl": 37.98503494262695, + "objective/non_score_reward": -1.8992516994476318, + "objective/rlhf_reward": 13.171060562133789, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007948282873257995, + "policy/clipfrac_avg": 0.002496684668585658, + "policy/entropy_avg": 0.1510624885559082, + "step": 936, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 3.4012387914117426e-05 + }, + { + "episode": 14992, + "epoch": 0.7571717171717172, + "eps": 0, + "loss/policy_avg": 0.1368754655122757, + "loss/value_avg": 0.6238852143287659, + "lr": 1.5408269478365883e-06, + "objective/entropy": -60.082557678222656, + "objective/kl": 41.95027160644531, + "objective/non_score_reward": -2.0975136756896973, + "objective/rlhf_reward": 12.652486801147461, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009200765052810311, + "policy/clipfrac_avg": 0.0059701185673475266, + "policy/entropy_avg": 0.17743873596191406, + "step": 937, + "val/clipfrac_avg": 0.012415202334523201, + "val/num_eos_tokens": 54, + "val/ratio": 0.9879815578460693, + "val/ratio_var": 4.387104490888305e-05 + }, + { + "episode": 15008, + "epoch": 0.757979797979798, + "eps": 0, + "loss/policy_avg": -0.17327642440795898, + "loss/value_avg": 0.538246750831604, + "lr": 1.5311947198572918e-06, + "objective/entropy": -112.668701171875, + "objective/kl": 35.57097625732422, + "objective/non_score_reward": -1.7785489559173584, + "objective/rlhf_reward": 13.416763305664062, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008795931353233755, + "policy/clipfrac_avg": 0.005144854076206684, + "policy/entropy_avg": 0.16583538055419922, + "step": 938, + "val/clipfrac_avg": 0.047972679138183594, + "val/num_eos_tokens": 136, + "val/ratio": 0.9902178645133972, + "val/ratio_var": 5.2144809160381556e-05 + }, + { + "episode": 15024, + "epoch": 0.7587878787878788, + "eps": 0, + "loss/policy_avg": 0.5920745134353638, + "loss/value_avg": 1.1476694345474243, + "lr": 1.5215872469825682e-06, + "objective/entropy": -72.79998779296875, + "objective/kl": 35.88062286376953, + "objective/non_score_reward": -1.7940311431884766, + "objective/rlhf_reward": 14.46378231048584, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008520716801285744, + "policy/clipfrac_avg": 0.0030313697643578053, + "policy/entropy_avg": 0.19948351383209229, + "step": 939, + "val/clipfrac_avg": 0.021162573248147964, + "val/num_eos_tokens": 210, + "val/ratio": 0.9882235527038574, + "val/ratio_var": 6.201990618137643e-05 + }, + { + "episode": 15040, + "epoch": 0.7595959595959596, + "eps": 0, + "loss/policy_avg": -0.15639176964759827, + "loss/value_avg": 0.8813156485557556, + "lr": 1.5120045977760784e-06, + "objective/entropy": -91.86679077148438, + "objective/kl": 42.35734176635742, + "objective/non_score_reward": -2.1178669929504395, + "objective/rlhf_reward": 15.264945983886719, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0010741858277469873, + "policy/clipfrac_avg": 0.00790454912930727, + "policy/entropy_avg": 0.19293498992919922, + "step": 940, + "val/clipfrac_avg": 0.015625, + "val/num_eos_tokens": 9, + "val/ratio": 0.986987829208374, + "val/ratio_var": 6.426944310078397e-05 + }, + { + "episode": 15056, + "epoch": 0.7604040404040404, + "eps": 0, + "loss/policy_avg": 0.2165047824382782, + "loss/value_avg": 0.34554335474967957, + "lr": 1.502446840624331e-06, + "objective/entropy": -62.31670379638672, + "objective/kl": 31.252155303955078, + "objective/non_score_reward": -1.562608003616333, + "objective/rlhf_reward": 14.156142234802246, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007184218848124146, + "policy/clipfrac_avg": 0.0035393601283431053, + "policy/entropy_avg": 0.14359378814697266, + "step": 941, + "val/clipfrac_avg": 0.013209432363510132, + "val/num_eos_tokens": 36, + "val/ratio": 0.9896091222763062, + "val/ratio_var": 5.599396536126733e-05 + }, + { + "episode": 15072, + "epoch": 0.7612121212121212, + "eps": 0, + "loss/policy_avg": -0.030075984075665474, + "loss/value_avg": 0.45760178565979004, + "lr": 1.4929140437361916e-06, + "objective/entropy": -109.5699691772461, + "objective/kl": 38.075103759765625, + "objective/non_score_reward": -1.9037551879882812, + "objective/rlhf_reward": 14.713432312011719, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010412151459604502, + "policy/clipfrac_avg": 0.009930627420544624, + "policy/entropy_avg": 0.18667316436767578, + "step": 942, + "val/clipfrac_avg": 0.014139890670776367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9863598346710205, + "val/ratio_var": 0.00010634912905516103 + }, + { + "episode": 15088, + "epoch": 0.762020202020202, + "eps": 0, + "loss/policy_avg": 0.12722979485988617, + "loss/value_avg": 0.5696362853050232, + "lr": 1.4834062751424018e-06, + "objective/entropy": -121.374267578125, + "objective/kl": 34.23678207397461, + "objective/non_score_reward": -1.711839199066162, + "objective/rlhf_reward": 14.038161277770996, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007155380444601178, + "policy/clipfrac_avg": 0.00446352269500494, + "policy/entropy_avg": 0.13922500610351562, + "step": 943, + "val/clipfrac_avg": 0.001972501166164875, + "val/num_eos_tokens": 493, + "val/ratio": 0.9906710386276245, + "val/ratio_var": 6.088761438149959e-05 + }, + { + "episode": 15104, + "epoch": 0.7628282828282829, + "eps": 0, + "loss/policy_avg": 0.24459479749202728, + "loss/value_avg": 0.997427225112915, + "lr": 1.4739236026950838e-06, + "objective/entropy": -117.67542266845703, + "objective/kl": 42.60675811767578, + "objective/non_score_reward": -2.130337953567505, + "objective/rlhf_reward": 13.854036331176758, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007362811593338847, + "policy/clipfrac_avg": 0.009282913990318775, + "policy/entropy_avg": 0.15163874626159668, + "step": 944, + "val/clipfrac_avg": 0.019384628161787987, + "val/num_eos_tokens": 404, + "val/ratio": 0.9907106161117554, + "val/ratio_var": 5.1745548262260854e-05 + }, + { + "episode": 15120, + "epoch": 0.7636363636363637, + "eps": 0, + "loss/policy_avg": 0.019024059176445007, + "loss/value_avg": 0.2505372166633606, + "lr": 1.4644660940672628e-06, + "objective/entropy": -72.49942779541016, + "objective/kl": 36.86358642578125, + "objective/non_score_reward": -1.8431792259216309, + "objective/rlhf_reward": 14.227133750915527, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009655736503191292, + "policy/clipfrac_avg": 0.0035610971972346306, + "policy/entropy_avg": 0.19275665283203125, + "step": 945, + "val/clipfrac_avg": 0.0008617835119366646, + "val/num_eos_tokens": 33, + "val/ratio": 0.9873601198196411, + "val/ratio_var": 7.99532572273165e-05 + }, + { + "episode": 15136, + "epoch": 0.7644444444444445, + "eps": 0, + "loss/policy_avg": -0.24745053052902222, + "loss/value_avg": 0.4052670896053314, + "lr": 1.455033816752383e-06, + "objective/entropy": -106.38670349121094, + "objective/kl": 30.419466018676758, + "objective/non_score_reward": -1.5209734439849854, + "objective/rlhf_reward": 15.338401794433594, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009057885035872459, + "policy/clipfrac_avg": 0.009060706943273544, + "policy/entropy_avg": 0.161224365234375, + "step": 946, + "val/clipfrac_avg": 0.038856737315654755, + "val/num_eos_tokens": 409, + "val/ratio": 0.9891325235366821, + "val/ratio_var": 9.022479935083538e-05 + }, + { + "episode": 15152, + "epoch": 0.7652525252525253, + "eps": 0, + "loss/policy_avg": -0.051016077399253845, + "loss/value_avg": 0.5337303876876831, + "lr": 1.4456268380638262e-06, + "objective/entropy": -104.14939880371094, + "objective/kl": 36.21892547607422, + "objective/non_score_reward": -1.8109462261199951, + "objective/rlhf_reward": 15.282804489135742, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009193423902615905, + "policy/clipfrac_avg": 0.010075472295284271, + "policy/entropy_avg": 0.16665363311767578, + "step": 947, + "val/clipfrac_avg": 0.006654549390077591, + "val/num_eos_tokens": 424, + "val/ratio": 0.9888213276863098, + "val/ratio_var": 8.222387987188995e-05 + }, + { + "episode": 15168, + "epoch": 0.7660606060606061, + "eps": 0, + "loss/policy_avg": 0.5708428025245667, + "loss/value_avg": 0.9156904220581055, + "lr": 1.4362452251344283e-06, + "objective/entropy": -107.8341064453125, + "objective/kl": 37.6370849609375, + "objective/non_score_reward": -1.8818542957305908, + "objective/rlhf_reward": 11.461895942687988, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008800949435681105, + "policy/clipfrac_avg": 0.018943525850772858, + "policy/entropy_avg": 0.18418407440185547, + "step": 948, + "val/clipfrac_avg": 0.0006793478387407959, + "val/num_eos_tokens": 297, + "val/ratio": 0.9880671501159668, + "val/ratio_var": 7.767622446408495e-05 + }, + { + "episode": 15184, + "epoch": 0.7668686868686869, + "eps": 0, + "loss/policy_avg": -0.028068287298083305, + "loss/value_avg": 0.6219231486320496, + "lr": 1.4268890449160034e-06, + "objective/entropy": -11.566868782043457, + "objective/kl": 34.966495513916016, + "objective/non_score_reward": -1.7483247518539429, + "objective/rlhf_reward": 14.603238105773926, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0011735728476196527, + "policy/clipfrac_avg": 0.004941962193697691, + "policy/entropy_avg": 0.20639419555664062, + "step": 949, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 32, + "val/ratio": 0.9847333431243896, + "val/ratio_var": 3.736758299055509e-05 + }, + { + "episode": 15200, + "epoch": 0.7676767676767676, + "eps": 0, + "loss/policy_avg": -0.15076535940170288, + "loss/value_avg": 0.2775721549987793, + "lr": 1.4175583641788698e-06, + "objective/entropy": -80.4765625, + "objective/kl": 31.678754806518555, + "objective/non_score_reward": -1.583937644958496, + "objective/rlhf_reward": 13.853562355041504, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0006830325582996011, + "policy/clipfrac_avg": 0.00417131744325161, + "policy/entropy_avg": 0.1293621063232422, + "step": 950, + "val/clipfrac_avg": 0.0393822118639946, + "val/num_eos_tokens": 257, + "val/ratio": 0.9911376237869263, + "val/ratio_var": 2.9909415388829075e-05 + }, + { + "episode": 15216, + "epoch": 0.7684848484848484, + "eps": 0, + "loss/policy_avg": -0.04640840366482735, + "loss/value_avg": 0.3922756314277649, + "lr": 1.4082532495113627e-06, + "objective/entropy": -84.18096160888672, + "objective/kl": 38.69972229003906, + "objective/non_score_reward": -1.9349862337112427, + "objective/rlhf_reward": 15.416576385498047, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.000798642635345459, + "policy/clipfrac_avg": 0.0038171219639480114, + "policy/entropy_avg": 0.1776995062828064, + "step": 951, + "val/clipfrac_avg": 0.0032731681130826473, + "val/num_eos_tokens": 326, + "val/ratio": 0.9889781475067139, + "val/ratio_var": 5.715083170798607e-05 + }, + { + "episode": 15232, + "epoch": 0.7692929292929293, + "eps": 0, + "loss/policy_avg": -0.12606917321681976, + "loss/value_avg": 0.3175104856491089, + "lr": 1.3989737673193682e-06, + "objective/entropy": -60.43904113769531, + "objective/kl": 39.91594314575195, + "objective/non_score_reward": -1.9957971572875977, + "objective/rlhf_reward": 14.262016296386719, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009779632091522217, + "policy/clipfrac_avg": 0.004585625603795052, + "policy/entropy_avg": 0.2008209228515625, + "step": 952, + "val/clipfrac_avg": 0.030662208795547485, + "val/num_eos_tokens": 422, + "val/ratio": 0.9862244129180908, + "val/ratio_var": 8.592133963247761e-05 + }, + { + "episode": 15248, + "epoch": 0.7701010101010101, + "eps": 0, + "loss/policy_avg": 0.08768834173679352, + "loss/value_avg": 0.327705055475235, + "lr": 1.389719983825844e-06, + "objective/entropy": -67.14826965332031, + "objective/kl": 46.73610305786133, + "objective/non_score_reward": -2.3368051052093506, + "objective/rlhf_reward": 14.29600715637207, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011253968114033341, + "policy/clipfrac_avg": 0.006955197546631098, + "policy/entropy_avg": 0.20795369148254395, + "step": 953, + "val/clipfrac_avg": 8.926542068365961e-05, + "val/num_eos_tokens": 421, + "val/ratio": 0.9854748249053955, + "val/ratio_var": 6.634320016019046e-05 + }, + { + "episode": 15264, + "epoch": 0.7709090909090909, + "eps": 0, + "loss/policy_avg": 0.17172515392303467, + "loss/value_avg": 0.5182530283927917, + "lr": 1.3804919650703551e-06, + "objective/entropy": -52.92948913574219, + "objective/kl": 39.47554016113281, + "objective/non_score_reward": -1.9737770557403564, + "objective/rlhf_reward": 15.323097229003906, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009001094149425626, + "policy/clipfrac_avg": 0.005291195120662451, + "policy/entropy_avg": 0.18181610107421875, + "step": 954, + "val/clipfrac_avg": 0.006615057587623596, + "val/num_eos_tokens": 490, + "val/ratio": 0.9876183271408081, + "val/ratio_var": 5.796940240543336e-05 + }, + { + "episode": 15280, + "epoch": 0.7717171717171717, + "eps": 0, + "loss/policy_avg": 0.3471048176288605, + "loss/value_avg": 0.7186154723167419, + "lr": 1.3712897769085903e-06, + "objective/entropy": -93.25178527832031, + "objective/kl": 41.423980712890625, + "objective/non_score_reward": -2.0711989402770996, + "objective/rlhf_reward": 13.155363082885742, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009469924843870103, + "policy/clipfrac_avg": 0.004164956044405699, + "policy/entropy_avg": 0.18822860717773438, + "step": 955, + "val/clipfrac_avg": 0.04515676945447922, + "val/num_eos_tokens": 611, + "val/ratio": 0.9877511262893677, + "val/ratio_var": 7.300172001123428e-05 + }, + { + "episode": 15296, + "epoch": 0.7725252525252525, + "eps": 0, + "loss/policy_avg": 0.24007092416286469, + "loss/value_avg": 0.4516885280609131, + "lr": 1.3621134850119012e-06, + "objective/entropy": -112.50715637207031, + "objective/kl": 31.20224380493164, + "objective/non_score_reward": -1.5601121187210083, + "objective/rlhf_reward": 13.799263000488281, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007952903397381306, + "policy/clipfrac_avg": 0.008523119613528252, + "policy/entropy_avg": 0.1457352638244629, + "step": 956, + "val/clipfrac_avg": 0.06015515699982643, + "val/num_eos_tokens": 383, + "val/ratio": 0.9895913600921631, + "val/ratio_var": 7.848602399462834e-05 + }, + { + "episode": 15312, + "epoch": 0.7733333333333333, + "eps": 0, + "loss/policy_avg": -0.03721753507852554, + "loss/value_avg": 0.241817444562912, + "lr": 1.3529631548668298e-06, + "objective/entropy": -82.6330337524414, + "objective/kl": 44.714515686035156, + "objective/non_score_reward": -2.2357258796691895, + "objective/rlhf_reward": 13.818962097167969, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0006799673428758979, + "policy/clipfrac_avg": 0.003063515294343233, + "policy/entropy_avg": 0.1481151580810547, + "step": 957, + "val/clipfrac_avg": 9.245562250725925e-05, + "val/num_eos_tokens": 341, + "val/ratio": 0.9905509948730469, + "val/ratio_var": 4.010691918665543e-05 + }, + { + "episode": 15328, + "epoch": 0.7741414141414141, + "eps": 0, + "loss/policy_avg": -0.1448572874069214, + "loss/value_avg": 0.6361722946166992, + "lr": 1.3438388517746431e-06, + "objective/entropy": -75.73651885986328, + "objective/kl": 32.26376724243164, + "objective/non_score_reward": -1.6131882667541504, + "objective/rlhf_reward": 13.550874710083008, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010009508114308119, + "policy/clipfrac_avg": 0.004912985023111105, + "policy/entropy_avg": 0.190643310546875, + "step": 958, + "val/clipfrac_avg": 0.008336715400218964, + "val/num_eos_tokens": 329, + "val/ratio": 0.9864659905433655, + "val/ratio_var": 9.685769327916205e-05 + }, + { + "episode": 15344, + "epoch": 0.7749494949494949, + "eps": 0, + "loss/policy_avg": 0.11857907474040985, + "loss/value_avg": 0.8868049383163452, + "lr": 1.3347406408508695e-06, + "objective/entropy": -90.25643920898438, + "objective/kl": 39.02738952636719, + "objective/non_score_reward": -1.9513695240020752, + "objective/rlhf_reward": 14.954880714416504, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009018229902721941, + "policy/clipfrac_avg": 0.004778256639838219, + "policy/entropy_avg": 0.1796565055847168, + "step": 959, + "val/clipfrac_avg": 0.02706507220864296, + "val/num_eos_tokens": 289, + "val/ratio": 0.9882634878158569, + "val/ratio_var": 9.323142148787156e-05 + }, + { + "episode": 15360, + "epoch": 0.7757575757575758, + "eps": 0, + "loss/policy_avg": -0.22503328323364258, + "loss/value_avg": 0.4810031056404114, + "lr": 1.3256685870248227e-06, + "objective/entropy": -101.29171752929688, + "objective/kl": 37.34764099121094, + "objective/non_score_reward": -1.8673819303512573, + "objective/rlhf_reward": 14.32011890411377, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008543966105207801, + "policy/clipfrac_avg": 0.0047050644643604755, + "policy/entropy_avg": 0.17150402069091797, + "step": 960, + "val/clipfrac_avg": 0.0005405173869803548, + "val/num_eos_tokens": 130, + "val/ratio": 0.9882644414901733, + "val/ratio_var": 6.539086461998522e-05 + }, + { + "episode": 15376, + "epoch": 0.7765656565656566, + "eps": 0, + "loss/policy_avg": -0.0476013720035553, + "loss/value_avg": 0.682388961315155, + "lr": 1.3166227550391542e-06, + "objective/entropy": -97.00421142578125, + "objective/kl": 42.212120056152344, + "objective/non_score_reward": -2.1106061935424805, + "objective/rlhf_reward": 13.99876880645752, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000946523156017065, + "policy/clipfrac_avg": 0.0059087686240673065, + "policy/entropy_avg": 0.19147014617919922, + "step": 961, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 224, + "val/ratio": 0.9874411225318909, + "val/ratio_var": 5.3472565923584625e-05 + }, + { + "episode": 15392, + "epoch": 0.7773737373737374, + "eps": 0, + "loss/policy_avg": -0.26426494121551514, + "loss/value_avg": 0.47233304381370544, + "lr": 1.307603209449378e-06, + "objective/entropy": -60.28771209716797, + "objective/kl": 39.21135330200195, + "objective/non_score_reward": -1.9605677127838135, + "objective/rlhf_reward": 15.351932525634766, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009600625489838421, + "policy/clipfrac_avg": 0.0035959878005087376, + "policy/entropy_avg": 0.18239498138427734, + "step": 962, + "val/clipfrac_avg": 0.012040323577821255, + "val/num_eos_tokens": 296, + "val/ratio": 0.9875339269638062, + "val/ratio_var": 6.485576159320772e-05 + }, + { + "episode": 15408, + "epoch": 0.7781818181818182, + "eps": 0, + "loss/policy_avg": 0.04519583284854889, + "loss/value_avg": 0.7595553398132324, + "lr": 1.298610014623423e-06, + "objective/entropy": -146.63661193847656, + "objective/kl": 38.37220001220703, + "objective/non_score_reward": -1.9186100959777832, + "objective/rlhf_reward": 13.354827880859375, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007201317930594087, + "policy/clipfrac_avg": 0.0022899312898516655, + "policy/entropy_avg": 0.1562885046005249, + "step": 963, + "val/clipfrac_avg": 8.292270649690181e-05, + "val/num_eos_tokens": 521, + "val/ratio": 0.9906989336013794, + "val/ratio_var": 6.185618985909969e-05 + }, + { + "episode": 15424, + "epoch": 0.778989898989899, + "eps": 0, + "loss/policy_avg": -0.14623457193374634, + "loss/value_avg": 0.35101252794265747, + "lr": 1.2896432347411553e-06, + "objective/entropy": -66.08819580078125, + "objective/kl": 36.48185348510742, + "objective/non_score_reward": -1.8240927457809448, + "objective/rlhf_reward": 14.261844635009766, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009832159848883748, + "policy/clipfrac_avg": 0.006735666189342737, + "policy/entropy_avg": 0.19070911407470703, + "step": 964, + "val/clipfrac_avg": 0.0030241934582591057, + "val/num_eos_tokens": 421, + "val/ratio": 0.9872285723686218, + "val/ratio_var": 7.909470878075808e-05 + }, + { + "episode": 15440, + "epoch": 0.7797979797979798, + "eps": 0, + "loss/policy_avg": -0.1669466346502304, + "loss/value_avg": 0.6373587846755981, + "lr": 1.2807029337939403e-06, + "objective/entropy": -48.76942825317383, + "objective/kl": 36.51701354980469, + "objective/non_score_reward": -1.8258507251739502, + "objective/rlhf_reward": 13.861648559570312, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.000836203049402684, + "policy/clipfrac_avg": 0.006686762906610966, + "policy/entropy_avg": 0.17627906799316406, + "step": 965, + "val/clipfrac_avg": 0.009943181648850441, + "val/num_eos_tokens": 11, + "val/ratio": 0.9892607927322388, + "val/ratio_var": 5.405569390859455e-05 + }, + { + "episode": 15456, + "epoch": 0.7806060606060606, + "eps": 0, + "loss/policy_avg": -0.005087921395897865, + "loss/value_avg": 0.2712212800979614, + "lr": 1.2717891755841722e-06, + "objective/entropy": -59.34945297241211, + "objective/kl": 42.5623893737793, + "objective/non_score_reward": -2.128119468688965, + "objective/rlhf_reward": 13.918755531311035, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008710323600098491, + "policy/clipfrac_avg": 0.010207775980234146, + "policy/entropy_avg": 0.1739978790283203, + "step": 966, + "val/clipfrac_avg": 0.010802468284964561, + "val/num_eos_tokens": 246, + "val/ratio": 0.9884668588638306, + "val/ratio_var": 4.567186624626629e-05 + }, + { + "episode": 15472, + "epoch": 0.7814141414141414, + "eps": 0, + "loss/policy_avg": -0.5036620497703552, + "loss/value_avg": 0.39513152837753296, + "lr": 1.2629020237248241e-06, + "objective/entropy": -136.59652709960938, + "objective/kl": 29.273542404174805, + "objective/non_score_reward": -1.463677167892456, + "objective/rlhf_reward": 14.083197593688965, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0007826394285075366, + "policy/clipfrac_avg": 0.006299240980297327, + "policy/entropy_avg": 0.15417519211769104, + "step": 967, + "val/clipfrac_avg": 0.03271849453449249, + "val/num_eos_tokens": 249, + "val/ratio": 0.9902065992355347, + "val/ratio_var": 8.734448783798143e-05 + }, + { + "episode": 15488, + "epoch": 0.7822222222222223, + "eps": 0, + "loss/policy_avg": -0.04834941402077675, + "loss/value_avg": 0.49886178970336914, + "lr": 1.2540415416389917e-06, + "objective/entropy": -29.31902503967285, + "objective/kl": 38.655548095703125, + "objective/non_score_reward": -1.9327776432037354, + "objective/rlhf_reward": 13.864096641540527, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008980755810625851, + "policy/clipfrac_avg": 0.003656789194792509, + "policy/entropy_avg": 0.1847667694091797, + "step": 968, + "val/clipfrac_avg": 5.3879310144111514e-05, + "val/num_eos_tokens": 10, + "val/ratio": 0.9879658818244934, + "val/ratio_var": 2.3956074073794298e-05 + }, + { + "episode": 15504, + "epoch": 0.7830303030303031, + "eps": 0, + "loss/policy_avg": -0.2217503786087036, + "loss/value_avg": 0.4100498557090759, + "lr": 1.2452077925594435e-06, + "objective/entropy": -206.95504760742188, + "objective/kl": 41.82144546508789, + "objective/non_score_reward": -2.0910723209381104, + "objective/rlhf_reward": 13.076896667480469, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008509804611094296, + "policy/clipfrac_avg": 0.0047798133455216885, + "policy/entropy_avg": 0.15673446655273438, + "step": 969, + "val/clipfrac_avg": 0.00021062050655018538, + "val/num_eos_tokens": 481, + "val/ratio": 0.9887630939483643, + "val/ratio_var": 6.604092777706683e-05 + }, + { + "episode": 15520, + "epoch": 0.7838383838383839, + "eps": 0, + "loss/policy_avg": -0.02120068296790123, + "loss/value_avg": 0.6901993751525879, + "lr": 1.23640083952817e-06, + "objective/entropy": -40.900238037109375, + "objective/kl": 38.07307052612305, + "objective/non_score_reward": -1.9036535024642944, + "objective/rlhf_reward": 14.377596855163574, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010727059561759233, + "policy/clipfrac_avg": 0.004337977152317762, + "policy/entropy_avg": 0.1984100341796875, + "step": 970, + "val/clipfrac_avg": 0.02858383022248745, + "val/num_eos_tokens": 535, + "val/ratio": 0.9863028526306152, + "val/ratio_var": 8.229557715822011e-05 + }, + { + "episode": 15536, + "epoch": 0.7846464646464646, + "eps": 0, + "loss/policy_avg": -0.09917282313108444, + "loss/value_avg": 0.3991038203239441, + "lr": 1.2276207453959283e-06, + "objective/entropy": -77.6534652709961, + "objective/kl": 39.35569381713867, + "objective/non_score_reward": -1.9677847623825073, + "objective/rlhf_reward": 13.907215118408203, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008703683270141482, + "policy/clipfrac_avg": 0.0043828776106238365, + "policy/entropy_avg": 0.17210960388183594, + "step": 971, + "val/clipfrac_avg": 0.00012400794366840273, + "val/num_eos_tokens": 175, + "val/ratio": 0.988074541091919, + "val/ratio_var": 5.318454350344837e-05 + }, + { + "episode": 15552, + "epoch": 0.7854545454545454, + "eps": 0, + "loss/policy_avg": 0.0814996063709259, + "loss/value_avg": 0.6175538897514343, + "lr": 1.2188675728217986e-06, + "objective/entropy": -16.92827606201172, + "objective/kl": 41.14687728881836, + "objective/non_score_reward": -2.0573439598083496, + "objective/rlhf_reward": 15.442655563354492, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010900306515395641, + "policy/clipfrac_avg": 0.00866631604731083, + "policy/entropy_avg": 0.2063446044921875, + "step": 972, + "val/clipfrac_avg": 0.009025804698467255, + "val/num_eos_tokens": 174, + "val/ratio": 0.9854716062545776, + "val/ratio_var": 2.7480951757752337e-05 + }, + { + "episode": 15568, + "epoch": 0.7862626262626262, + "eps": 0, + "loss/policy_avg": 0.3593114912509918, + "loss/value_avg": 0.7372395396232605, + "lr": 1.2101413842727345e-06, + "objective/entropy": -63.132266998291016, + "objective/kl": 45.04676818847656, + "objective/non_score_reward": -2.252338409423828, + "objective/rlhf_reward": 14.450786590576172, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009616734459996223, + "policy/clipfrac_avg": 0.0046102688647806644, + "policy/entropy_avg": 0.1856832504272461, + "step": 973, + "val/clipfrac_avg": 0.016889449208974838, + "val/num_eos_tokens": 419, + "val/ratio": 0.9877485036849976, + "val/ratio_var": 6.443963502533734e-05 + }, + { + "episode": 15584, + "epoch": 0.787070707070707, + "eps": 0, + "loss/policy_avg": 0.14888805150985718, + "loss/value_avg": 0.7544107437133789, + "lr": 1.201442242023122e-06, + "objective/entropy": -98.38099670410156, + "objective/kl": 41.9028434753418, + "objective/non_score_reward": -2.095142126083374, + "objective/rlhf_reward": 12.186107635498047, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009328118758276105, + "policy/clipfrac_avg": 0.004906289279460907, + "policy/entropy_avg": 0.17488479614257812, + "step": 974, + "val/clipfrac_avg": 0.08914318680763245, + "val/num_eos_tokens": 193, + "val/ratio": 0.9888035655021667, + "val/ratio_var": 6.763465353287756e-05 + }, + { + "episode": 15600, + "epoch": 0.7878787878787878, + "eps": 0, + "loss/policy_avg": -0.05416979640722275, + "loss/value_avg": 0.5626612901687622, + "lr": 1.1927702081543279e-06, + "objective/entropy": -65.54731750488281, + "objective/kl": 33.80537796020508, + "objective/non_score_reward": -1.6902689933776855, + "objective/rlhf_reward": 13.434730529785156, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010051055578514934, + "policy/clipfrac_avg": 0.005396673921495676, + "policy/entropy_avg": 0.19644975662231445, + "step": 975, + "val/clipfrac_avg": 0.02293882891535759, + "val/num_eos_tokens": 539, + "val/ratio": 0.9861206412315369, + "val/ratio_var": 7.48153543099761e-05 + }, + { + "episode": 15616, + "epoch": 0.7886868686868687, + "eps": 0, + "loss/policy_avg": 0.05607307702302933, + "loss/value_avg": 0.48446062207221985, + "lr": 1.1841253445542605e-06, + "objective/entropy": -29.001220703125, + "objective/kl": 36.37082290649414, + "objective/non_score_reward": -1.8185412883758545, + "objective/rlhf_reward": 12.142396926879883, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0011397147318348289, + "policy/clipfrac_avg": 0.0031259057577699423, + "policy/entropy_avg": 0.22170257568359375, + "step": 976, + "val/clipfrac_avg": 0.005424283444881439, + "val/num_eos_tokens": 387, + "val/ratio": 0.9855947494506836, + "val/ratio_var": 6.33225790807046e-05 + }, + { + "episode": 15632, + "epoch": 0.7894949494949495, + "eps": 0, + "loss/policy_avg": -0.15400168299674988, + "loss/value_avg": 0.8165977001190186, + "lr": 1.1755077129169284e-06, + "objective/entropy": -60.56960678100586, + "objective/kl": 37.47486114501953, + "objective/non_score_reward": -1.8737430572509766, + "objective/rlhf_reward": 11.046178817749023, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0009815995581448078, + "policy/clipfrac_avg": 0.005213981028646231, + "policy/entropy_avg": 0.21601486206054688, + "step": 977, + "val/clipfrac_avg": 0.022167012095451355, + "val/num_eos_tokens": 93, + "val/ratio": 0.9861747026443481, + "val/ratio_var": 7.86496966611594e-05 + }, + { + "episode": 15648, + "epoch": 0.7903030303030303, + "eps": 0, + "loss/policy_avg": -0.1725654900074005, + "loss/value_avg": 0.487238347530365, + "lr": 1.166917374742e-06, + "objective/entropy": -56.807586669921875, + "objective/kl": 37.225948333740234, + "objective/non_score_reward": -1.861297607421875, + "objective/rlhf_reward": 14.388702392578125, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102808338124305, + "policy/clipfrac_avg": 0.0037505230866372585, + "policy/entropy_avg": 0.20328998565673828, + "step": 978, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 128, + "val/ratio": 0.9871898293495178, + "val/ratio_var": 9.372083150083199e-05 + }, + { + "episode": 15664, + "epoch": 0.7911111111111111, + "eps": 0, + "loss/policy_avg": -0.2764543294906616, + "loss/value_avg": 0.33185577392578125, + "lr": 1.158354391334362e-06, + "objective/entropy": -78.30638122558594, + "objective/kl": 34.59331512451172, + "objective/non_score_reward": -1.729665756225586, + "objective/rlhf_reward": 15.489084243774414, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008904373971745372, + "policy/clipfrac_avg": 0.00305290799587965, + "policy/entropy_avg": 0.17419052124023438, + "step": 979, + "val/clipfrac_avg": 0.09470640122890472, + "val/num_eos_tokens": 367, + "val/ratio": 0.9881394505500793, + "val/ratio_var": 6.588637916138396e-05 + }, + { + "episode": 15680, + "epoch": 0.7919191919191919, + "eps": 0, + "loss/policy_avg": 0.049546245485544205, + "loss/value_avg": 0.5992924571037292, + "lr": 1.1498188238036862e-06, + "objective/entropy": -146.6639862060547, + "objective/kl": 40.94211959838867, + "objective/non_score_reward": -2.0471062660217285, + "objective/rlhf_reward": 12.40992546081543, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008619745494797826, + "policy/clipfrac_avg": 0.007621705532073975, + "policy/entropy_avg": 0.15411758422851562, + "step": 980, + "val/clipfrac_avg": 0.09697519987821579, + "val/num_eos_tokens": 405, + "val/ratio": 0.9889684915542603, + "val/ratio_var": 5.157575287739746e-05 + }, + { + "episode": 15696, + "epoch": 0.7927272727272727, + "eps": 0, + "loss/policy_avg": 0.7101428508758545, + "loss/value_avg": 1.0345678329467773, + "lr": 1.141310733063991e-06, + "objective/entropy": -111.55574798583984, + "objective/kl": 35.79730224609375, + "objective/non_score_reward": -1.7898651361465454, + "objective/rlhf_reward": 13.764822006225586, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009224195964634418, + "policy/clipfrac_avg": 0.00732937827706337, + "policy/entropy_avg": 0.1737055778503418, + "step": 981, + "val/clipfrac_avg": 0.0015625000232830644, + "val/num_eos_tokens": 230, + "val/ratio": 0.9885663390159607, + "val/ratio_var": 7.395897409878671e-05 + }, + { + "episode": 15712, + "epoch": 0.7935353535353535, + "eps": 0, + "loss/policy_avg": -0.1941525638103485, + "loss/value_avg": 0.4167664647102356, + "lr": 1.132830179833207e-06, + "objective/entropy": -136.7578125, + "objective/kl": 41.060279846191406, + "objective/non_score_reward": -2.053014039993286, + "objective/rlhf_reward": 14.564172744750977, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008562670554965734, + "policy/clipfrac_avg": 0.005737176164984703, + "policy/entropy_avg": 0.1706523895263672, + "step": 982, + "val/clipfrac_avg": 0.006640523672103882, + "val/num_eos_tokens": 935, + "val/ratio": 0.9893268346786499, + "val/ratio_var": 6.982601917115971e-05 + }, + { + "episode": 15728, + "epoch": 0.7943434343434344, + "eps": 0, + "loss/policy_avg": -0.1428111046552658, + "loss/value_avg": 0.2174670249223709, + "lr": 1.1243772246327416e-06, + "objective/entropy": -109.03129577636719, + "objective/kl": 38.72102355957031, + "objective/non_score_reward": -1.9360511302947998, + "objective/rlhf_reward": 14.470198631286621, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000771217979490757, + "policy/clipfrac_avg": 0.002998888725414872, + "policy/entropy_avg": 0.16396713256835938, + "step": 983, + "val/clipfrac_avg": 0.020510124042630196, + "val/num_eos_tokens": 25, + "val/ratio": 0.989021897315979, + "val/ratio_var": 4.0093269490171224e-05 + }, + { + "episode": 15744, + "epoch": 0.7951515151515152, + "eps": 0, + "loss/policy_avg": 0.010445185005664825, + "loss/value_avg": 0.37888824939727783, + "lr": 1.1159519277870507e-06, + "objective/entropy": -6.635685920715332, + "objective/kl": 38.89892578125, + "objective/non_score_reward": -1.9449464082717896, + "objective/rlhf_reward": 14.7113037109375, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009513833792880177, + "policy/clipfrac_avg": 0.003157425671815872, + "policy/entropy_avg": 0.1973400115966797, + "step": 984, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 21, + "val/ratio": 0.9875931739807129, + "val/ratio_var": 5.873597910976969e-05 + }, + { + "episode": 15760, + "epoch": 0.795959595959596, + "eps": 0, + "loss/policy_avg": -0.28500208258628845, + "loss/value_avg": 0.4213254451751709, + "lr": 1.1075543494232049e-06, + "objective/entropy": -149.3587646484375, + "objective/kl": 31.575908660888672, + "objective/non_score_reward": -1.5787954330444336, + "objective/rlhf_reward": 15.75714111328125, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007162499241530895, + "policy/clipfrac_avg": 0.0078894617035985, + "policy/entropy_avg": 0.14484691619873047, + "step": 985, + "val/clipfrac_avg": 0.023666253313422203, + "val/num_eos_tokens": 155, + "val/ratio": 0.9908950328826904, + "val/ratio_var": 5.517086901818402e-05 + }, + { + "episode": 15776, + "epoch": 0.7967676767676768, + "eps": 0, + "loss/policy_avg": -0.016947008669376373, + "loss/value_avg": 0.3959222733974457, + "lr": 1.0991845494704673e-06, + "objective/entropy": -90.30593872070312, + "objective/kl": 44.143898010253906, + "objective/non_score_reward": -2.2071948051452637, + "objective/rlhf_reward": 12.292805671691895, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000890448980499059, + "policy/clipfrac_avg": 0.005722125992178917, + "policy/entropy_avg": 0.1587371826171875, + "step": 986, + "val/clipfrac_avg": 0.020597470924258232, + "val/num_eos_tokens": 327, + "val/ratio": 0.9887195825576782, + "val/ratio_var": 5.094557855045423e-05 + }, + { + "episode": 15792, + "epoch": 0.7975757575757576, + "eps": 0, + "loss/policy_avg": 0.022888287901878357, + "loss/value_avg": 0.36602234840393066, + "lr": 1.0908425876598512e-06, + "objective/entropy": -140.4935760498047, + "objective/kl": 42.647743225097656, + "objective/non_score_reward": -2.132387161254883, + "objective/rlhf_reward": 13.516050338745117, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008821362862363458, + "policy/clipfrac_avg": 0.005631787236779928, + "policy/entropy_avg": 0.1700587272644043, + "step": 987, + "val/clipfrac_avg": 0.012090040370821953, + "val/num_eos_tokens": 546, + "val/ratio": 0.9886399507522583, + "val/ratio_var": 5.682192204403691e-05 + }, + { + "episode": 15808, + "epoch": 0.7983838383838384, + "eps": 0, + "loss/policy_avg": 0.19011469185352325, + "loss/value_avg": 0.5114719271659851, + "lr": 1.082528523523713e-06, + "objective/entropy": -53.488460540771484, + "objective/kl": 36.984073638916016, + "objective/non_score_reward": -1.8492038249969482, + "objective/rlhf_reward": 14.049233436584473, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009789207251742482, + "policy/clipfrac_avg": 0.005884348414838314, + "policy/entropy_avg": 0.20690059661865234, + "step": 988, + "val/clipfrac_avg": 0.013353430666029453, + "val/num_eos_tokens": 353, + "val/ratio": 0.9872037768363953, + "val/ratio_var": 6.569022662006319e-05 + }, + { + "episode": 15824, + "epoch": 0.7991919191919192, + "eps": 0, + "loss/policy_avg": 0.1753331869840622, + "loss/value_avg": 0.3773725628852844, + "lr": 1.0742424163953096e-06, + "objective/entropy": -56.68682861328125, + "objective/kl": 37.34782409667969, + "objective/non_score_reward": -1.8673913478851318, + "objective/rlhf_reward": 13.050577163696289, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.000838876876514405, + "policy/clipfrac_avg": 0.0064134905114769936, + "policy/entropy_avg": 0.16259098052978516, + "step": 989, + "val/clipfrac_avg": 0.055415935814380646, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885395169258118, + "val/ratio_var": 8.253273699665442e-05 + }, + { + "episode": 15840, + "epoch": 0.8, + "eps": 0, + "loss/policy_avg": -0.013544011861085892, + "loss/value_avg": 0.5490726828575134, + "lr": 1.0659843254083919e-06, + "objective/entropy": -84.58714294433594, + "objective/kl": 34.28784942626953, + "objective/non_score_reward": -1.7143926620483398, + "objective/rlhf_reward": 13.20748233795166, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009669119026511908, + "policy/clipfrac_avg": 0.00355747202411294, + "policy/entropy_avg": 0.18613052368164062, + "step": 990, + "val/clipfrac_avg": 0.015838205814361572, + "val/num_eos_tokens": 115, + "val/ratio": 0.9877301454544067, + "val/ratio_var": 5.967232209513895e-05 + }, + { + "episode": 15856, + "epoch": 0.8008080808080809, + "eps": 0, + "loss/policy_avg": 0.17707303166389465, + "loss/value_avg": 0.24937717616558075, + "lr": 1.0577543094967613e-06, + "objective/entropy": -134.7545928955078, + "objective/kl": 35.57737731933594, + "objective/non_score_reward": -1.7788686752319336, + "objective/rlhf_reward": 15.010193824768066, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008520459523424506, + "policy/clipfrac_avg": 0.007036273833364248, + "policy/entropy_avg": 0.14983177185058594, + "step": 991, + "val/clipfrac_avg": 0.0005040322430431843, + "val/num_eos_tokens": 450, + "val/ratio": 0.9898406863212585, + "val/ratio_var": 4.8867601435631514e-05 + }, + { + "episode": 15872, + "epoch": 0.8016161616161617, + "eps": 0, + "loss/policy_avg": -0.1231779083609581, + "loss/value_avg": 0.27385565638542175, + "lr": 1.0495524273938735e-06, + "objective/entropy": -99.90908813476562, + "objective/kl": 37.30508804321289, + "objective/non_score_reward": -1.865254521369934, + "objective/rlhf_reward": 12.759745597839355, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008709601243026555, + "policy/clipfrac_avg": 0.004131123423576355, + "policy/entropy_avg": 0.17008209228515625, + "step": 992, + "val/clipfrac_avg": 0.01071952749043703, + "val/num_eos_tokens": 376, + "val/ratio": 0.9894505739212036, + "val/ratio_var": 6.905445479787886e-05 + }, + { + "episode": 15888, + "epoch": 0.8024242424242424, + "eps": 0, + "loss/policy_avg": -0.11878392100334167, + "loss/value_avg": 0.6012347340583801, + "lr": 1.041378737632402e-06, + "objective/entropy": -89.06427764892578, + "objective/kl": 39.19951629638672, + "objective/non_score_reward": -1.9599759578704834, + "objective/rlhf_reward": 14.532211303710938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009216031758114696, + "policy/clipfrac_avg": 0.00672511849552393, + "policy/entropy_avg": 0.1765270233154297, + "step": 993, + "val/clipfrac_avg": 0.0031250000465661287, + "val/num_eos_tokens": 74, + "val/ratio": 0.9882692098617554, + "val/ratio_var": 3.506496432237327e-05 + }, + { + "episode": 15904, + "epoch": 0.8032323232323232, + "eps": 0, + "loss/policy_avg": 0.11932457238435745, + "loss/value_avg": 0.28720468282699585, + "lr": 1.0332332985438248e-06, + "objective/entropy": -48.50525665283203, + "objective/kl": 37.94336700439453, + "objective/non_score_reward": -1.8971682786941528, + "objective/rlhf_reward": 15.641894340515137, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010036650346592069, + "policy/clipfrac_avg": 0.0061247204430401325, + "policy/entropy_avg": 0.18225669860839844, + "step": 994, + "val/clipfrac_avg": 0.01019021775573492, + "val/num_eos_tokens": 8, + "val/ratio": 0.9874961972236633, + "val/ratio_var": 5.3736544941784814e-05 + }, + { + "episode": 15920, + "epoch": 0.804040404040404, + "eps": 0, + "loss/policy_avg": 0.14462614059448242, + "loss/value_avg": 0.3247252106666565, + "lr": 1.0251161682580125e-06, + "objective/entropy": -108.57736206054688, + "objective/kl": 41.1889762878418, + "objective/non_score_reward": -2.059448719024658, + "objective/rlhf_reward": 14.182738304138184, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001018329057842493, + "policy/clipfrac_avg": 0.0034168935380876064, + "policy/entropy_avg": 0.20872879028320312, + "step": 995, + "val/clipfrac_avg": 0.00548538425937295, + "val/num_eos_tokens": 8, + "val/ratio": 0.9865425825119019, + "val/ratio_var": 8.348035044036806e-05 + }, + { + "episode": 15936, + "epoch": 0.8048484848484848, + "eps": 0, + "loss/policy_avg": 0.2817779779434204, + "loss/value_avg": 0.49794071912765503, + "lr": 1.0170274047028068e-06, + "objective/entropy": -43.08762741088867, + "objective/kl": 45.49590301513672, + "objective/non_score_reward": -2.2747950553894043, + "objective/rlhf_reward": 13.912704467773438, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010373364202678204, + "policy/clipfrac_avg": 0.006608482450246811, + "policy/entropy_avg": 0.2166609764099121, + "step": 996, + "val/clipfrac_avg": 0.03890983760356903, + "val/num_eos_tokens": 11, + "val/ratio": 0.986150860786438, + "val/ratio_var": 8.553406951250508e-05 + }, + { + "episode": 15952, + "epoch": 0.8056565656565656, + "eps": 0, + "loss/policy_avg": -0.22047999501228333, + "loss/value_avg": 0.5745018720626831, + "lr": 1.0089670656036137e-06, + "objective/entropy": -114.1541976928711, + "objective/kl": 35.899574279785156, + "objective/non_score_reward": -1.7949787378311157, + "objective/rlhf_reward": 14.869083404541016, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007367507787421346, + "policy/clipfrac_avg": 0.0037586798425763845, + "policy/entropy_avg": 0.15613555908203125, + "step": 997, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 560, + "val/ratio": 0.9902479648590088, + "val/ratio_var": 5.6676559324841946e-05 + }, + { + "episode": 15968, + "epoch": 0.8064646464646464, + "eps": 0, + "loss/policy_avg": -0.04391951486468315, + "loss/value_avg": 0.6516869068145752, + "lr": 1.0009352084829871e-06, + "objective/entropy": -112.42098999023438, + "objective/kl": 39.436988830566406, + "objective/non_score_reward": -1.9718495607376099, + "objective/rlhf_reward": 14.48127555847168, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009028672939166427, + "policy/clipfrac_avg": 0.0059207589365541935, + "policy/entropy_avg": 0.18039512634277344, + "step": 998, + "val/clipfrac_avg": 0.0127427838742733, + "val/num_eos_tokens": 515, + "val/ratio": 0.9878066778182983, + "val/ratio_var": 8.427277498412877e-05 + }, + { + "episode": 15984, + "epoch": 0.8072727272727273, + "eps": 0, + "loss/policy_avg": -0.5726251602172852, + "loss/value_avg": 0.6322129964828491, + "lr": 9.929318906602176e-07, + "objective/entropy": -115.74237060546875, + "objective/kl": 35.111915588378906, + "objective/non_score_reward": -1.7555959224700928, + "objective/rlhf_reward": 14.103778839111328, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006586643867194653, + "policy/clipfrac_avg": 0.006481423042714596, + "policy/entropy_avg": 0.1394972801208496, + "step": 999, + "val/clipfrac_avg": 0.019941631704568863, + "val/num_eos_tokens": 81, + "val/ratio": 0.9906902313232422, + "val/ratio_var": 5.2608244004659355e-05 + }, + { + "episode": 16000, + "epoch": 0.8080808080808081, + "eps": 0, + "loss/policy_avg": -0.1571139395236969, + "loss/value_avg": 0.8596901297569275, + "lr": 9.849571692509257e-07, + "objective/entropy": -35.66822052001953, + "objective/kl": 36.119667053222656, + "objective/non_score_reward": -1.805983304977417, + "objective/rlhf_reward": 12.842453956604004, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008497819071635604, + "policy/clipfrac_avg": 0.0041199941188097, + "policy/entropy_avg": 0.16556644439697266, + "step": 1000, + "val/clipfrac_avg": 0.022714650258421898, + "val/num_eos_tokens": 82, + "val/ratio": 0.9879409074783325, + "val/ratio_var": 6.342474807752296e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3bc835020acd1e5226b4380fb2120e50bc1de2ba --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:642a94cfb7b053d6e2a7bb2cbf7c2e8727588482facf77b9edc31830a6555ea3 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3fe84faaf0e9b61977a3c86d5c988286135b3c63 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/trainer_state.json @@ -0,0 +1,23135 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 17600, + "epoch": 0.8888888888888888, + "eval_steps": 100, + "global_step": 1100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + }, + { + "episode": 11216, + "epoch": 0.5664646464646464, + "eps": 0, + "loss/policy_avg": -0.09055965393781662, + "loss/value_avg": 0.5455071926116943, + "lr": 4.334128543718295e-06, + "objective/entropy": -74.95745849609375, + "objective/kl": 33.42132568359375, + "objective/non_score_reward": -1.6710662841796875, + "objective/rlhf_reward": 14.828933715820312, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0013184514828026295, + "policy/clipfrac_avg": 0.00809475127607584, + "policy/entropy_avg": 0.17902374267578125, + "step": 701, + "val/clipfrac_avg": 0.18833743035793304, + "val/num_eos_tokens": 358, + "val/ratio": 0.9879262447357178, + "val/ratio_var": 6.120221951277927e-05 + }, + { + "episode": 11232, + "epoch": 0.5672727272727273, + "eps": 0, + "loss/policy_avg": 0.2261451780796051, + "loss/value_avg": 1.5403046607971191, + "lr": 4.320892800523827e-06, + "objective/entropy": -90.61648559570312, + "objective/kl": 27.537485122680664, + "objective/non_score_reward": -1.3768744468688965, + "objective/rlhf_reward": 12.826250076293945, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009536159923300147, + "policy/clipfrac_avg": 0.01545755285769701, + "policy/entropy_avg": 0.20562219619750977, + "step": 702, + "val/clipfrac_avg": 0.033450622111558914, + "val/num_eos_tokens": 595, + "val/ratio": 0.9873440265655518, + "val/ratio_var": 7.613677007611841e-05 + }, + { + "episode": 11248, + "epoch": 0.5680808080808081, + "eps": 0, + "loss/policy_avg": -0.05153674632310867, + "loss/value_avg": 0.18954747915267944, + "lr": 4.307661903773129e-06, + "objective/entropy": -56.119659423828125, + "objective/kl": 43.216548919677734, + "objective/non_score_reward": -2.160827398300171, + "objective/rlhf_reward": 15.80792236328125, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010682737920433283, + "policy/clipfrac_avg": 0.005114596337080002, + "policy/entropy_avg": 0.20781707763671875, + "step": 703, + "val/clipfrac_avg": 0.12178023159503937, + "val/num_eos_tokens": 244, + "val/ratio": 0.9860707521438599, + "val/ratio_var": 5.392924140323885e-05 + }, + { + "episode": 11264, + "epoch": 0.5688888888888889, + "eps": 0, + "loss/policy_avg": 0.22105136513710022, + "loss/value_avg": 0.43452760577201843, + "lr": 4.294435947888405e-06, + "objective/entropy": -180.00924682617188, + "objective/kl": 32.27890396118164, + "objective/non_score_reward": -1.6139451265335083, + "objective/rlhf_reward": 12.425117492675781, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007389815291389823, + "policy/clipfrac_avg": 0.00613048393279314, + "policy/entropy_avg": 0.15887287259101868, + "step": 704, + "val/clipfrac_avg": 0.12273523956537247, + "val/num_eos_tokens": 611, + "val/ratio": 0.9900252819061279, + "val/ratio_var": 6.185931124491617e-05 + }, + { + "episode": 11280, + "epoch": 0.5696969696969697, + "eps": 0, + "loss/policy_avg": -0.044020622968673706, + "loss/value_avg": 1.459969401359558, + "lr": 4.281215027256592e-06, + "objective/entropy": -73.2503890991211, + "objective/kl": 29.207317352294922, + "objective/non_score_reward": -1.460365891456604, + "objective/rlhf_reward": 15.766195297241211, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010228066239506006, + "policy/clipfrac_avg": 0.010731351561844349, + "policy/entropy_avg": 0.17015647888183594, + "step": 705, + "val/clipfrac_avg": 0.1418685019016266, + "val/num_eos_tokens": 378, + "val/ratio": 0.9865360856056213, + "val/ratio_var": 0.0001155980207840912 + }, + { + "episode": 11296, + "epoch": 0.5705050505050505, + "eps": 0, + "loss/policy_avg": 0.10734500735998154, + "loss/value_avg": 0.40052735805511475, + "lr": 4.2679992362287e-06, + "objective/entropy": -91.05632019042969, + "objective/kl": 39.329261779785156, + "objective/non_score_reward": -1.966463327407837, + "objective/rlhf_reward": 12.744474411010742, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010600072564557195, + "policy/clipfrac_avg": 0.005786027293652296, + "policy/entropy_avg": 0.23768234252929688, + "step": 706, + "val/clipfrac_avg": 0.032500892877578735, + "val/num_eos_tokens": 203, + "val/ratio": 0.9860942959785461, + "val/ratio_var": 5.682241317117587e-05 + }, + { + "episode": 11312, + "epoch": 0.5713131313131313, + "eps": 0, + "loss/policy_avg": -0.5062837600708008, + "loss/value_avg": 0.7742470502853394, + "lr": 4.254788669119127e-06, + "objective/entropy": -93.2535400390625, + "objective/kl": 34.540130615234375, + "objective/non_score_reward": -1.727006435394287, + "objective/rlhf_reward": 14.788618087768555, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008475214708596468, + "policy/clipfrac_avg": 0.027025574818253517, + "policy/entropy_avg": 0.21597671508789062, + "step": 707, + "val/clipfrac_avg": 0.1775030940771103, + "val/num_eos_tokens": 165, + "val/ratio": 0.9894334077835083, + "val/ratio_var": 9.222733206115663e-05 + }, + { + "episode": 11328, + "epoch": 0.5721212121212121, + "eps": 0, + "loss/policy_avg": -0.1463669091463089, + "loss/value_avg": 0.5632686614990234, + "lr": 4.241583420204998e-06, + "objective/entropy": -110.37503051757812, + "objective/kl": 36.62909698486328, + "objective/non_score_reward": -1.831454873085022, + "objective/rlhf_reward": 12.89510726928711, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009251610608771443, + "policy/clipfrac_avg": 0.00941617414355278, + "policy/entropy_avg": 0.16801166534423828, + "step": 708, + "val/clipfrac_avg": 0.04514690116047859, + "val/num_eos_tokens": 267, + "val/ratio": 0.9897239208221436, + "val/ratio_var": 7.692311191931367e-05 + }, + { + "episode": 11344, + "epoch": 0.572929292929293, + "eps": 0, + "loss/policy_avg": -0.35295569896698, + "loss/value_avg": 0.7035583257675171, + "lr": 4.228383583725471e-06, + "objective/entropy": -76.27204132080078, + "objective/kl": 31.900005340576172, + "objective/non_score_reward": -1.5950002670288086, + "objective/rlhf_reward": 14.053436279296875, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010020419722422957, + "policy/clipfrac_avg": 0.011507621966302395, + "policy/entropy_avg": 0.16324424743652344, + "step": 709, + "val/clipfrac_avg": 0.12508229911327362, + "val/num_eos_tokens": 173, + "val/ratio": 0.9896982908248901, + "val/ratio_var": 4.9723090342013165e-05 + }, + { + "episode": 11360, + "epoch": 0.5737373737373738, + "eps": 0, + "loss/policy_avg": 0.160404771566391, + "loss/value_avg": 1.1914963722229004, + "lr": 4.215189253881093e-06, + "objective/entropy": -79.55085754394531, + "objective/kl": 36.81852722167969, + "objective/non_score_reward": -1.8409266471862793, + "objective/rlhf_reward": 12.159073829650879, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0011080782860517502, + "policy/clipfrac_avg": 0.006503967102617025, + "policy/entropy_avg": 0.19612884521484375, + "step": 710, + "val/clipfrac_avg": 0.08140315115451813, + "val/num_eos_tokens": 538, + "val/ratio": 0.9866147041320801, + "val/ratio_var": 7.869482942624018e-05 + }, + { + "episode": 11376, + "epoch": 0.5745454545454546, + "eps": 0, + "loss/policy_avg": 0.07548795640468597, + "loss/value_avg": 0.5138013362884521, + "lr": 4.2020005248331056e-06, + "objective/entropy": -119.03164672851562, + "objective/kl": 34.31198501586914, + "objective/non_score_reward": -1.7155992984771729, + "objective/rlhf_reward": 13.503150939941406, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007362160831689835, + "policy/clipfrac_avg": 0.00589254405349493, + "policy/entropy_avg": 0.14674091339111328, + "step": 711, + "val/clipfrac_avg": 0.02668866142630577, + "val/num_eos_tokens": 515, + "val/ratio": 0.990146279335022, + "val/ratio_var": 3.9673428545938805e-05 + }, + { + "episode": 11392, + "epoch": 0.5753535353535354, + "eps": 0, + "loss/policy_avg": -0.2925986051559448, + "loss/value_avg": 0.3821415603160858, + "lr": 4.1888174907027775e-06, + "objective/entropy": -112.06816101074219, + "objective/kl": 38.006500244140625, + "objective/non_score_reward": -1.900325059890747, + "objective/rlhf_reward": 13.990300178527832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.000874843040946871, + "policy/clipfrac_avg": 0.007544378750026226, + "policy/entropy_avg": 0.16875195503234863, + "step": 712, + "val/clipfrac_avg": 0.025153353810310364, + "val/num_eos_tokens": 4, + "val/ratio": 0.9893897771835327, + "val/ratio_var": 8.519255788996816e-05 + }, + { + "episode": 11408, + "epoch": 0.5761616161616162, + "eps": 0, + "loss/policy_avg": -0.5444605350494385, + "loss/value_avg": 1.517474889755249, + "lr": 4.175640245570742e-06, + "objective/entropy": -92.28114318847656, + "objective/kl": 35.69511413574219, + "objective/non_score_reward": -1.7847557067871094, + "objective/rlhf_reward": 13.73868179321289, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008441185345873237, + "policy/clipfrac_avg": 0.005670716054737568, + "policy/entropy_avg": 0.1591033935546875, + "step": 713, + "val/clipfrac_avg": 0.1791025698184967, + "val/num_eos_tokens": 98, + "val/ratio": 0.9903852939605713, + "val/ratio_var": 4.043114677187987e-05 + }, + { + "episode": 11424, + "epoch": 0.576969696969697, + "eps": 0, + "loss/policy_avg": -0.3059786260128021, + "loss/value_avg": 0.6795393228530884, + "lr": 4.162468883476319e-06, + "objective/entropy": -193.77468872070312, + "objective/kl": 42.17440414428711, + "objective/non_score_reward": -2.108720302581787, + "objective/rlhf_reward": 12.281905174255371, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008298080647364259, + "policy/clipfrac_avg": 0.00635810662060976, + "policy/entropy_avg": 0.15823698043823242, + "step": 714, + "val/clipfrac_avg": 0.0064142923802137375, + "val/num_eos_tokens": 1666, + "val/ratio": 0.9892975091934204, + "val/ratio_var": 6.228621350601315e-05 + }, + { + "episode": 11440, + "epoch": 0.5777777777777777, + "eps": 0, + "loss/policy_avg": -0.023257479071617126, + "loss/value_avg": 0.18199321627616882, + "lr": 4.149303498416838e-06, + "objective/entropy": -22.63348388671875, + "objective/kl": 44.308345794677734, + "objective/non_score_reward": -2.2154173851013184, + "objective/rlhf_reward": 14.82364559173584, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0012546201469376683, + "policy/clipfrac_avg": 0.007735411170870066, + "policy/entropy_avg": 0.240936279296875, + "step": 715, + "val/clipfrac_avg": 0.0028473599813878536, + "val/num_eos_tokens": 302, + "val/ratio": 0.9840776920318604, + "val/ratio_var": 5.949061960563995e-05 + }, + { + "episode": 11456, + "epoch": 0.5785858585858585, + "eps": 0, + "loss/policy_avg": 0.3237575888633728, + "loss/value_avg": 0.38197559118270874, + "lr": 4.136144184346979e-06, + "objective/entropy": -168.4746856689453, + "objective/kl": 40.63113784790039, + "objective/non_score_reward": -2.031557083129883, + "objective/rlhf_reward": 13.070005416870117, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0007934231543913484, + "policy/clipfrac_avg": 0.012050900608301163, + "policy/entropy_avg": 0.14413070678710938, + "step": 716, + "val/clipfrac_avg": 0.09322839975357056, + "val/num_eos_tokens": 1174, + "val/ratio": 0.9900047779083252, + "val/ratio_var": 5.29003118572291e-05 + }, + { + "episode": 11472, + "epoch": 0.5793939393939394, + "eps": 0, + "loss/policy_avg": 0.23654325306415558, + "loss/value_avg": 1.166879653930664, + "lr": 4.122991035178093e-06, + "objective/entropy": -96.77055358886719, + "objective/kl": 42.17526626586914, + "objective/non_score_reward": -2.1087632179260254, + "objective/rlhf_reward": 14.500611305236816, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000815618084743619, + "policy/clipfrac_avg": 0.008080050349235535, + "policy/entropy_avg": 0.14806270599365234, + "step": 717, + "val/clipfrac_avg": 0.02973807230591774, + "val/num_eos_tokens": 297, + "val/ratio": 0.9899397492408752, + "val/ratio_var": 3.871042645187117e-05 + }, + { + "episode": 11488, + "epoch": 0.5802020202020202, + "eps": 0, + "loss/policy_avg": -0.11968575417995453, + "loss/value_avg": 0.9426589608192444, + "lr": 4.109844144777537e-06, + "objective/entropy": -58.205047607421875, + "objective/kl": 42.6419792175293, + "objective/non_score_reward": -2.132099151611328, + "objective/rlhf_reward": 12.203838348388672, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009566791704855859, + "policy/clipfrac_avg": 0.0025268401950597763, + "policy/entropy_avg": 0.18893003463745117, + "step": 718, + "val/clipfrac_avg": 0.1098824143409729, + "val/num_eos_tokens": 16, + "val/ratio": 0.9869214296340942, + "val/ratio_var": 8.384165266761556e-05 + }, + { + "episode": 11504, + "epoch": 0.581010101010101, + "eps": 0, + "loss/policy_avg": 0.0208362378180027, + "loss/value_avg": 0.8499202728271484, + "lr": 4.096703606968007e-06, + "objective/entropy": -105.71292114257812, + "objective/kl": 40.70527648925781, + "objective/non_score_reward": -2.035263776779175, + "objective/rlhf_reward": 13.605360984802246, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008132618968375027, + "policy/clipfrac_avg": 0.004117591306567192, + "policy/entropy_avg": 0.15980243682861328, + "step": 719, + "val/clipfrac_avg": 0.04368842393159866, + "val/num_eos_tokens": 428, + "val/ratio": 0.9895676374435425, + "val/ratio_var": 6.342094275169075e-05 + }, + { + "episode": 11520, + "epoch": 0.5818181818181818, + "eps": 0, + "loss/policy_avg": -0.11627452820539474, + "loss/value_avg": 0.7368168830871582, + "lr": 4.083569515526853e-06, + "objective/entropy": -100.23733520507812, + "objective/kl": 32.93721008300781, + "objective/non_score_reward": -1.6468604803085327, + "objective/rlhf_reward": 15.134389877319336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008385346736758947, + "policy/clipfrac_avg": 0.009182492271065712, + "policy/entropy_avg": 0.17890548706054688, + "step": 720, + "val/clipfrac_avg": 0.06867726147174835, + "val/num_eos_tokens": 144, + "val/ratio": 0.9896779656410217, + "val/ratio_var": 7.192800694610924e-05 + }, + { + "episode": 11536, + "epoch": 0.5826262626262626, + "eps": 0, + "loss/policy_avg": -0.06581735610961914, + "loss/value_avg": 0.7933100461959839, + "lr": 4.070441964185428e-06, + "objective/entropy": -103.3985366821289, + "objective/kl": 39.985321044921875, + "objective/non_score_reward": -1.9992659091949463, + "objective/rlhf_reward": 14.547609329223633, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000881620217114687, + "policy/clipfrac_avg": 0.004334163852035999, + "policy/entropy_avg": 0.1855926513671875, + "step": 721, + "val/clipfrac_avg": 0.04114583134651184, + "val/num_eos_tokens": 161, + "val/ratio": 0.989449679851532, + "val/ratio_var": 6.215622852323577e-05 + }, + { + "episode": 11552, + "epoch": 0.5834343434343434, + "eps": 0, + "loss/policy_avg": 0.44041353464126587, + "loss/value_avg": 0.7198789119720459, + "lr": 4.05732104662841e-06, + "objective/entropy": -119.77040100097656, + "objective/kl": 36.42585754394531, + "objective/non_score_reward": -1.8212929964065552, + "objective/rlhf_reward": 13.467769622802734, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008070225012488663, + "policy/clipfrac_avg": 0.007022568956017494, + "policy/entropy_avg": 0.15318524837493896, + "step": 722, + "val/clipfrac_avg": 0.05163312703371048, + "val/num_eos_tokens": 304, + "val/ratio": 0.9899815320968628, + "val/ratio_var": 4.889031333732419e-05 + }, + { + "episode": 11568, + "epoch": 0.5842424242424242, + "eps": 0, + "loss/policy_avg": 0.4866091012954712, + "loss/value_avg": 1.1953067779541016, + "lr": 4.04420685649314e-06, + "objective/entropy": -50.46599578857422, + "objective/kl": 39.113037109375, + "objective/non_score_reward": -1.955651879310608, + "objective/rlhf_reward": 13.911535263061523, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010366745991632342, + "policy/clipfrac_avg": 0.006084335036575794, + "policy/entropy_avg": 0.20149445533752441, + "step": 723, + "val/clipfrac_avg": 0.09489050507545471, + "val/num_eos_tokens": 36, + "val/ratio": 0.9867548942565918, + "val/ratio_var": 6.938407750567421e-05 + }, + { + "episode": 11584, + "epoch": 0.585050505050505, + "eps": 0, + "loss/policy_avg": 0.1589544266462326, + "loss/value_avg": 0.568170964717865, + "lr": 4.031099487368937e-06, + "objective/entropy": -112.03081512451172, + "objective/kl": 37.355255126953125, + "objective/non_score_reward": -1.8677629232406616, + "objective/rlhf_reward": 13.49161148071289, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001035548048093915, + "policy/clipfrac_avg": 0.006865138188004494, + "policy/entropy_avg": 0.20564031600952148, + "step": 724, + "val/clipfrac_avg": 0.05233292281627655, + "val/num_eos_tokens": 347, + "val/ratio": 0.9859846830368042, + "val/ratio_var": 0.00010150864545721561 + }, + { + "episode": 11600, + "epoch": 0.5858585858585859, + "eps": 0, + "loss/policy_avg": -0.19635428488254547, + "loss/value_avg": 0.33023789525032043, + "lr": 4.0179990327964526e-06, + "objective/entropy": -69.79931640625, + "objective/kl": 41.90481948852539, + "objective/non_score_reward": -2.095241069793701, + "objective/rlhf_reward": 14.85007095336914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0010360007872805, + "policy/clipfrac_avg": 0.004560987465083599, + "policy/entropy_avg": 0.20290184020996094, + "step": 725, + "val/clipfrac_avg": 0.08375490456819534, + "val/num_eos_tokens": 261, + "val/ratio": 0.98649001121521, + "val/ratio_var": 6.110723916208372e-05 + }, + { + "episode": 11616, + "epoch": 0.5866666666666667, + "eps": 0, + "loss/policy_avg": 0.0999169647693634, + "loss/value_avg": 1.0625457763671875, + "lr": 4.004905586266988e-06, + "objective/entropy": -104.75285339355469, + "objective/kl": 32.01334762573242, + "objective/non_score_reward": -1.6006674766540527, + "objective/rlhf_reward": 12.766520500183105, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010034965816885233, + "policy/clipfrac_avg": 0.007322699762880802, + "policy/entropy_avg": 0.19988441467285156, + "step": 726, + "val/clipfrac_avg": 0.14562752842903137, + "val/num_eos_tokens": 33, + "val/ratio": 0.9887305498123169, + "val/ratio_var": 7.558859942946583e-05 + }, + { + "episode": 11632, + "epoch": 0.5874747474747475, + "eps": 0, + "loss/policy_avg": 0.004431502893567085, + "loss/value_avg": 0.5651569366455078, + "lr": 3.991819241221836e-06, + "objective/entropy": -53.65510940551758, + "objective/kl": 43.23419952392578, + "objective/non_score_reward": -2.161710262298584, + "objective/rlhf_reward": 12.986727714538574, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001010584644973278, + "policy/clipfrac_avg": 0.006680442951619625, + "policy/entropy_avg": 0.17978286743164062, + "step": 727, + "val/clipfrac_avg": 0.028638603165745735, + "val/num_eos_tokens": 25, + "val/ratio": 0.9880765676498413, + "val/ratio_var": 3.508182999212295e-05 + }, + { + "episode": 11648, + "epoch": 0.5882828282828283, + "eps": 0, + "loss/policy_avg": -0.026804931461811066, + "loss/value_avg": 0.3243325352668762, + "lr": 3.978740091051599e-06, + "objective/entropy": -147.60580444335938, + "objective/kl": 40.000404357910156, + "objective/non_score_reward": -2.0000202655792236, + "objective/rlhf_reward": 13.304666519165039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008220034651458263, + "policy/clipfrac_avg": 0.010559421963989735, + "policy/entropy_avg": 0.15192937850952148, + "step": 728, + "val/clipfrac_avg": 0.047058261930942535, + "val/num_eos_tokens": 93, + "val/ratio": 0.9890267252922058, + "val/ratio_var": 7.159873348427936e-05 + }, + { + "episode": 11664, + "epoch": 0.5890909090909091, + "eps": 0, + "loss/policy_avg": 0.061702877283096313, + "loss/value_avg": 0.25130608677864075, + "lr": 3.965668229095546e-06, + "objective/entropy": -75.26506042480469, + "objective/kl": 34.32489013671875, + "objective/non_score_reward": -1.7162445783615112, + "objective/rlhf_reward": 13.9556303024292, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008753975853323936, + "policy/clipfrac_avg": 0.006253514904528856, + "policy/entropy_avg": 0.17108154296875, + "step": 729, + "val/clipfrac_avg": 0.03564208000898361, + "val/num_eos_tokens": 194, + "val/ratio": 0.9883392453193665, + "val/ratio_var": 5.439678716356866e-05 + }, + { + "episode": 11680, + "epoch": 0.5898989898989899, + "eps": 0, + "loss/policy_avg": 0.5118429064750671, + "loss/value_avg": 0.3809865713119507, + "lr": 3.9526037486409275e-06, + "objective/entropy": -211.5462646484375, + "objective/kl": 38.28752136230469, + "objective/non_score_reward": -1.9143760204315186, + "objective/rlhf_reward": 13.062187194824219, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008200224256142974, + "policy/clipfrac_avg": 0.009272422641515732, + "policy/entropy_avg": 0.15618228912353516, + "step": 730, + "val/clipfrac_avg": 0.13058164715766907, + "val/num_eos_tokens": 191, + "val/ratio": 0.9892928600311279, + "val/ratio_var": 5.910802065045573e-05 + }, + { + "episode": 11696, + "epoch": 0.5907070707070707, + "eps": 0, + "loss/policy_avg": -0.575583279132843, + "loss/value_avg": 0.6588011384010315, + "lr": 3.939546742922318e-06, + "objective/entropy": -155.975341796875, + "objective/kl": 33.536251068115234, + "objective/non_score_reward": -1.6768126487731934, + "objective/rlhf_reward": 14.198186874389648, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006373925716616213, + "policy/clipfrac_avg": 0.011547036468982697, + "policy/entropy_avg": 0.15375041961669922, + "step": 731, + "val/clipfrac_avg": 0.1309666782617569, + "val/num_eos_tokens": 393, + "val/ratio": 0.9912899732589722, + "val/ratio_var": 7.494341116398573e-05 + }, + { + "episode": 11712, + "epoch": 0.5915151515151515, + "eps": 0, + "loss/policy_avg": 0.08975643664598465, + "loss/value_avg": 0.4319133758544922, + "lr": 3.926497305120943e-06, + "objective/entropy": -46.366912841796875, + "objective/kl": 38.771400451660156, + "objective/non_score_reward": -1.9385700225830078, + "objective/rlhf_reward": 14.631742477416992, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010276802349835634, + "policy/clipfrac_avg": 0.004419120494276285, + "policy/entropy_avg": 0.19952774047851562, + "step": 732, + "val/clipfrac_avg": 0.02601846680045128, + "val/num_eos_tokens": 121, + "val/ratio": 0.9856929779052734, + "val/ratio_var": 5.2173218136886135e-05 + }, + { + "episode": 11728, + "epoch": 0.5923232323232324, + "eps": 0, + "loss/policy_avg": -0.5671601295471191, + "loss/value_avg": 1.4009994268417358, + "lr": 3.913455528364027e-06, + "objective/entropy": -30.65240478515625, + "objective/kl": 34.644386291503906, + "objective/non_score_reward": -1.7322194576263428, + "objective/rlhf_reward": 16.127155303955078, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009229708230122924, + "policy/clipfrac_avg": 0.0053136469796299934, + "policy/entropy_avg": 0.1684565544128418, + "step": 733, + "val/clipfrac_avg": 0.16779303550720215, + "val/num_eos_tokens": 257, + "val/ratio": 0.98807692527771, + "val/ratio_var": 4.935808829031885e-05 + }, + { + "episode": 11744, + "epoch": 0.5931313131313132, + "eps": 0, + "loss/policy_avg": 0.1117314025759697, + "loss/value_avg": 0.6027213335037231, + "lr": 3.900421505724119e-06, + "objective/entropy": -115.830078125, + "objective/kl": 38.92997741699219, + "objective/non_score_reward": -1.9464988708496094, + "objective/rlhf_reward": 12.38162612915039, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006995117873884737, + "policy/clipfrac_avg": 0.0067988173104822636, + "policy/entropy_avg": 0.12786293029785156, + "step": 734, + "val/clipfrac_avg": 0.11902365833520889, + "val/num_eos_tokens": 256, + "val/ratio": 0.9913206696510315, + "val/ratio_var": 3.536290387273766e-05 + }, + { + "episode": 11760, + "epoch": 0.593939393939394, + "eps": 0, + "loss/policy_avg": 0.07707646489143372, + "loss/value_avg": 0.8557188510894775, + "lr": 3.887395330218429e-06, + "objective/entropy": -77.05795288085938, + "objective/kl": 34.79220199584961, + "objective/non_score_reward": -1.739610195159912, + "objective/rlhf_reward": 12.510390281677246, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009019911522045732, + "policy/clipfrac_avg": 0.00438590906560421, + "policy/entropy_avg": 0.18432998657226562, + "step": 735, + "val/clipfrac_avg": 0.20910227298736572, + "val/num_eos_tokens": 339, + "val/ratio": 0.9879709482192993, + "val/ratio_var": 7.31311592971906e-05 + }, + { + "episode": 11776, + "epoch": 0.5947474747474748, + "eps": 0, + "loss/policy_avg": 0.005743637681007385, + "loss/value_avg": 0.4823000729084015, + "lr": 3.8743770948081675e-06, + "objective/entropy": -189.58908081054688, + "objective/kl": 40.491024017333984, + "objective/non_score_reward": -2.0245513916015625, + "objective/rlhf_reward": 12.608261108398438, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010324950562790036, + "policy/clipfrac_avg": 0.008821255527436733, + "policy/entropy_avg": 0.1864786148071289, + "step": 736, + "val/clipfrac_avg": 0.03615584969520569, + "val/num_eos_tokens": 146, + "val/ratio": 0.9867680072784424, + "val/ratio_var": 0.00012203787628095597 + }, + { + "episode": 11792, + "epoch": 0.5955555555555555, + "eps": 0, + "loss/policy_avg": 0.11549818515777588, + "loss/value_avg": 0.4191131591796875, + "lr": 3.861366892397878e-06, + "objective/entropy": -53.57340621948242, + "objective/kl": 47.2681999206543, + "objective/non_score_reward": -2.363409996032715, + "objective/rlhf_reward": 11.238153457641602, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0010416920995339751, + "policy/clipfrac_avg": 0.006328020244836807, + "policy/entropy_avg": 0.21573257446289062, + "step": 737, + "val/clipfrac_avg": 0.05454040318727493, + "val/num_eos_tokens": 145, + "val/ratio": 0.9863517880439758, + "val/ratio_var": 5.17384396516718e-05 + }, + { + "episode": 11808, + "epoch": 0.5963636363636363, + "eps": 0, + "loss/policy_avg": 0.09252296388149261, + "loss/value_avg": 0.24368464946746826, + "lr": 3.848364815834782e-06, + "objective/entropy": -9.551445007324219, + "objective/kl": 41.81903839111328, + "objective/non_score_reward": -2.090951919555664, + "objective/rlhf_reward": 15.362173080444336, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.001077502267435193, + "policy/clipfrac_avg": 0.007244379259645939, + "policy/entropy_avg": 0.208465576171875, + "step": 738, + "val/clipfrac_avg": 0.046727634966373444, + "val/num_eos_tokens": 70, + "val/ratio": 0.9859708547592163, + "val/ratio_var": 4.752089807880111e-05 + }, + { + "episode": 11824, + "epoch": 0.5971717171717171, + "eps": 0, + "loss/policy_avg": -0.47144564986228943, + "loss/value_avg": 0.9659825563430786, + "lr": 3.835370957908108e-06, + "objective/entropy": -109.4366226196289, + "objective/kl": 32.577239990234375, + "objective/non_score_reward": -1.6288621425628662, + "objective/rlhf_reward": 13.332075119018555, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009311667527072132, + "policy/clipfrac_avg": 0.007746369112282991, + "policy/entropy_avg": 0.2024822235107422, + "step": 739, + "val/clipfrac_avg": 0.1542586386203766, + "val/num_eos_tokens": 140, + "val/ratio": 0.9888983964920044, + "val/ratio_var": 8.097994577838108e-05 + }, + { + "episode": 11840, + "epoch": 0.597979797979798, + "eps": 0, + "loss/policy_avg": 0.3934873938560486, + "loss/value_avg": 0.5160516500473022, + "lr": 3.822385411348429e-06, + "objective/entropy": -82.36312103271484, + "objective/kl": 38.616180419921875, + "objective/non_score_reward": -1.9308089017868042, + "objective/rlhf_reward": 12.725440979003906, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008991706417873502, + "policy/clipfrac_avg": 0.01843995228409767, + "policy/entropy_avg": 0.2245502471923828, + "step": 740, + "val/clipfrac_avg": 0.0067659346386790276, + "val/num_eos_tokens": 30, + "val/ratio": 0.988348662853241, + "val/ratio_var": 5.281575795379467e-05 + }, + { + "episode": 11856, + "epoch": 0.5987878787878788, + "eps": 0, + "loss/policy_avg": -0.06747608631849289, + "loss/value_avg": 0.3114325702190399, + "lr": 3.809408268827009e-06, + "objective/entropy": -89.22055053710938, + "objective/kl": 46.57951736450195, + "objective/non_score_reward": -2.3289759159088135, + "objective/rlhf_reward": 13.483524322509766, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000996086047962308, + "policy/clipfrac_avg": 0.005896539893001318, + "policy/entropy_avg": 0.19659423828125, + "step": 741, + "val/clipfrac_avg": 0.0699368417263031, + "val/num_eos_tokens": 107, + "val/ratio": 0.9875314235687256, + "val/ratio_var": 4.793483094545081e-05 + }, + { + "episode": 11872, + "epoch": 0.5995959595959596, + "eps": 0, + "loss/policy_avg": -0.2737995684146881, + "loss/value_avg": 0.4772384762763977, + "lr": 3.7964396229551365e-06, + "objective/entropy": -104.62603759765625, + "objective/kl": 36.626800537109375, + "objective/non_score_reward": -1.831339955329895, + "objective/rlhf_reward": 14.778034210205078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008760478813201189, + "policy/clipfrac_avg": 0.005249073728919029, + "policy/entropy_avg": 0.17531394958496094, + "step": 742, + "val/clipfrac_avg": 0.06328597664833069, + "val/num_eos_tokens": 91, + "val/ratio": 0.9885821342468262, + "val/ratio_var": 5.911157131777145e-05 + }, + { + "episode": 11888, + "epoch": 0.6004040404040404, + "eps": 0, + "loss/policy_avg": -0.11605305969715118, + "loss/value_avg": 0.30427810549736023, + "lr": 3.783479566283457e-06, + "objective/entropy": -123.75762176513672, + "objective/kl": 35.787086486816406, + "objective/non_score_reward": -1.7893544435501099, + "objective/rlhf_reward": 14.86689567565918, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007770034717395902, + "policy/clipfrac_avg": 0.0056176697835326195, + "policy/entropy_avg": 0.152313232421875, + "step": 743, + "val/clipfrac_avg": 0.048395052552223206, + "val/num_eos_tokens": 386, + "val/ratio": 0.9905653595924377, + "val/ratio_var": 5.709798642783426e-05 + }, + { + "episode": 11904, + "epoch": 0.6012121212121212, + "eps": 0, + "loss/policy_avg": -0.02294028550386429, + "loss/value_avg": 0.7253668904304504, + "lr": 3.7705281913013286e-06, + "objective/entropy": -39.3416633605957, + "objective/kl": 43.74962615966797, + "objective/non_score_reward": -2.18748140335083, + "objective/rlhf_reward": 12.648456573486328, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010895703453570604, + "policy/clipfrac_avg": 0.0052008070051670074, + "policy/entropy_avg": 0.22774124145507812, + "step": 744, + "val/clipfrac_avg": 0.01829145848751068, + "val/num_eos_tokens": 71, + "val/ratio": 0.985338568687439, + "val/ratio_var": 6.19284255662933e-05 + }, + { + "episode": 11920, + "epoch": 0.602020202020202, + "eps": 0, + "loss/policy_avg": 0.1328388750553131, + "loss/value_avg": 1.3390746116638184, + "lr": 3.7575855904361447e-06, + "objective/entropy": -79.38550567626953, + "objective/kl": 33.780616760253906, + "objective/non_score_reward": -1.6890308856964111, + "objective/rlhf_reward": 14.178156852722168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008661438478156924, + "policy/clipfrac_avg": 0.003558866214007139, + "policy/entropy_avg": 0.15964603424072266, + "step": 745, + "val/clipfrac_avg": 0.05471283197402954, + "val/num_eos_tokens": 81, + "val/ratio": 0.9897934198379517, + "val/ratio_var": 8.215933485189453e-05 + }, + { + "episode": 11936, + "epoch": 0.6028282828282828, + "eps": 0, + "loss/policy_avg": -0.29737263917922974, + "loss/value_avg": 0.6314270496368408, + "lr": 3.7446518560526903e-06, + "objective/entropy": -124.25365447998047, + "objective/kl": 42.908660888671875, + "objective/non_score_reward": -2.145432949066162, + "objective/rlhf_reward": 11.87019157409668, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009470224613323808, + "policy/clipfrac_avg": 0.006878838874399662, + "policy/entropy_avg": 0.1818103790283203, + "step": 746, + "val/clipfrac_avg": 0.0798804834485054, + "val/num_eos_tokens": 329, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 8.261724724434316e-05 + }, + { + "episode": 11952, + "epoch": 0.6036363636363636, + "eps": 0, + "loss/policy_avg": 0.12454540282487869, + "loss/value_avg": 0.4487236738204956, + "lr": 3.731727080452464e-06, + "objective/entropy": -72.31340026855469, + "objective/kl": 43.510276794433594, + "objective/non_score_reward": -2.175513982772827, + "objective/rlhf_reward": 12.347923278808594, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009941391181200743, + "policy/clipfrac_avg": 0.00658874586224556, + "policy/entropy_avg": 0.16569900512695312, + "step": 747, + "val/clipfrac_avg": 0.1540203094482422, + "val/num_eos_tokens": 77, + "val/ratio": 0.9888790845870972, + "val/ratio_var": 2.4041321012191474e-05 + }, + { + "episode": 11968, + "epoch": 0.6044444444444445, + "eps": 0, + "loss/policy_avg": -0.1547861397266388, + "loss/value_avg": 0.49022629857063293, + "lr": 3.7188113558730403e-06, + "objective/entropy": -103.35200500488281, + "objective/kl": 35.28123474121094, + "objective/non_score_reward": -1.764061689376831, + "objective/rlhf_reward": 13.27500057220459, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008191847009584308, + "policy/clipfrac_avg": 0.006343110464513302, + "policy/entropy_avg": 0.14904117584228516, + "step": 748, + "val/clipfrac_avg": 0.04430587589740753, + "val/num_eos_tokens": 124, + "val/ratio": 0.9908789396286011, + "val/ratio_var": 4.545380579656921e-05 + }, + { + "episode": 11984, + "epoch": 0.6052525252525253, + "eps": 0, + "loss/policy_avg": 0.11601711809635162, + "loss/value_avg": 0.6175556182861328, + "lr": 3.705904774487396e-06, + "objective/entropy": -111.66493225097656, + "objective/kl": 36.76349639892578, + "objective/non_score_reward": -1.838174819946289, + "objective/rlhf_reward": 13.693075180053711, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008918631356209517, + "policy/clipfrac_avg": 0.010910662822425365, + "policy/entropy_avg": 0.16454505920410156, + "step": 749, + "val/clipfrac_avg": 0.06755826622247696, + "val/num_eos_tokens": 358, + "val/ratio": 0.9886500835418701, + "val/ratio_var": 6.255816697375849e-05 + }, + { + "episode": 12000, + "epoch": 0.6060606060606061, + "eps": 0, + "loss/policy_avg": -0.04627024754881859, + "loss/value_avg": 0.6224689483642578, + "lr": 3.6930074284032613e-06, + "objective/entropy": -63.2913703918457, + "objective/kl": 32.27269744873047, + "objective/non_score_reward": -1.6136349439620972, + "objective/rlhf_reward": 14.644177436828613, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009490256779827178, + "policy/clipfrac_avg": 0.0068245502188801765, + "policy/entropy_avg": 0.17229938507080078, + "step": 750, + "val/clipfrac_avg": 0.18630267679691315, + "val/num_eos_tokens": 11, + "val/ratio": 0.9878020882606506, + "val/ratio_var": 5.455148493638262e-05 + }, + { + "episode": 12016, + "epoch": 0.6068686868686869, + "eps": 0, + "loss/policy_avg": -0.17890788614749908, + "loss/value_avg": 0.7140947580337524, + "lr": 3.6801194096624515e-06, + "objective/entropy": -54.77946472167969, + "objective/kl": 34.44614028930664, + "objective/non_score_reward": -1.7223069667816162, + "objective/rlhf_reward": 14.894880294799805, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011779924388974905, + "policy/clipfrac_avg": 0.024402525275945663, + "policy/entropy_avg": 0.2098255157470703, + "step": 751, + "val/clipfrac_avg": 0.04182715713977814, + "val/num_eos_tokens": 42, + "val/ratio": 0.9854939579963684, + "val/ratio_var": 0.00014133901277091354 + }, + { + "episode": 12032, + "epoch": 0.6076767676767677, + "eps": 0, + "loss/policy_avg": -0.0015007667243480682, + "loss/value_avg": 0.2544102668762207, + "lr": 3.6672408102402247e-06, + "objective/entropy": -70.776611328125, + "objective/kl": 37.67994689941406, + "objective/non_score_reward": -1.8839974403381348, + "objective/rlhf_reward": 12.108190536499023, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009373503271490335, + "policy/clipfrac_avg": 0.005210112780332565, + "policy/entropy_avg": 0.18710041046142578, + "step": 752, + "val/clipfrac_avg": 0.021128622815012932, + "val/num_eos_tokens": 218, + "val/ratio": 0.9874061942100525, + "val/ratio_var": 5.212551332078874e-05 + }, + { + "episode": 12048, + "epoch": 0.6084848484848485, + "eps": 0, + "loss/policy_avg": 0.05351484939455986, + "loss/value_avg": 0.28849858045578003, + "lr": 3.654371722044616e-06, + "objective/entropy": -89.17417907714844, + "objective/kl": 44.5589714050293, + "objective/non_score_reward": -2.2279486656188965, + "objective/rlhf_reward": 12.178300857543945, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009375615045428276, + "policy/clipfrac_avg": 0.005882620811462402, + "policy/entropy_avg": 0.18878555297851562, + "step": 753, + "val/clipfrac_avg": 0.021566759794950485, + "val/num_eos_tokens": 120, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 5.727734605898149e-05 + }, + { + "episode": 12064, + "epoch": 0.6092929292929293, + "eps": 0, + "loss/policy_avg": -0.12424173951148987, + "loss/value_avg": 0.359388530254364, + "lr": 3.641512236915783e-06, + "objective/entropy": -114.3979263305664, + "objective/kl": 43.28697967529297, + "objective/non_score_reward": -2.16434907913208, + "objective/rlhf_reward": 14.226276397705078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000866133370436728, + "policy/clipfrac_avg": 0.0050315093249082565, + "policy/entropy_avg": 0.16837501525878906, + "step": 754, + "val/clipfrac_avg": 0.1426960825920105, + "val/num_eos_tokens": 391, + "val/ratio": 0.9894628524780273, + "val/ratio_var": 5.016153590986505e-05 + }, + { + "episode": 12080, + "epoch": 0.6101010101010101, + "eps": 0, + "loss/policy_avg": -0.2821425795555115, + "loss/value_avg": 0.38819342851638794, + "lr": 3.6286624466253496e-06, + "objective/entropy": -164.57290649414062, + "objective/kl": 38.83340072631836, + "objective/non_score_reward": -1.9416700601577759, + "objective/rlhf_reward": 14.245829582214355, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008140793070197105, + "policy/clipfrac_avg": 0.00435154652222991, + "policy/entropy_avg": 0.15729975700378418, + "step": 755, + "val/clipfrac_avg": 0.09525477886199951, + "val/num_eos_tokens": 336, + "val/ratio": 0.989446222782135, + "val/ratio_var": 5.3185020078672096e-05 + }, + { + "episode": 12096, + "epoch": 0.610909090909091, + "eps": 0, + "loss/policy_avg": 0.05094696208834648, + "loss/value_avg": 0.4260476529598236, + "lr": 3.6158224428757538e-06, + "objective/entropy": -61.176116943359375, + "objective/kl": 32.80990219116211, + "objective/non_score_reward": -1.6404950618743896, + "objective/rlhf_reward": 16.24231719970703, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009597806492820382, + "policy/clipfrac_avg": 0.0061972904950380325, + "policy/entropy_avg": 0.16566753387451172, + "step": 756, + "val/clipfrac_avg": 0.01815173029899597, + "val/num_eos_tokens": 163, + "val/ratio": 0.9894044399261475, + "val/ratio_var": 4.711053770733997e-05 + }, + { + "episode": 12112, + "epoch": 0.6117171717171718, + "eps": 0, + "loss/policy_avg": -0.040469832718372345, + "loss/value_avg": 0.3746616244316101, + "lr": 3.602992317299594e-06, + "objective/entropy": -52.30070877075195, + "objective/kl": 34.218990325927734, + "objective/non_score_reward": -1.7109496593475342, + "objective/rlhf_reward": 13.273425102233887, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009450222714804113, + "policy/clipfrac_avg": 0.0049097794108092785, + "policy/entropy_avg": 0.1805877685546875, + "step": 757, + "val/clipfrac_avg": 0.09231511503458023, + "val/num_eos_tokens": 105, + "val/ratio": 0.9877369403839111, + "val/ratio_var": 4.407649248605594e-05 + }, + { + "episode": 12128, + "epoch": 0.6125252525252525, + "eps": 0, + "loss/policy_avg": -0.0790981650352478, + "loss/value_avg": 0.289002388715744, + "lr": 3.5901721614589725e-06, + "objective/entropy": -33.08611297607422, + "objective/kl": 42.89472961425781, + "objective/non_score_reward": -2.1447365283966064, + "objective/rlhf_reward": 14.792763710021973, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011628612410277128, + "policy/clipfrac_avg": 0.00601824838668108, + "policy/entropy_avg": 0.21850204467773438, + "step": 758, + "val/clipfrac_avg": 0.05997534096240997, + "val/num_eos_tokens": 208, + "val/ratio": 0.9858998656272888, + "val/ratio_var": 4.6025765186641365e-05 + }, + { + "episode": 12144, + "epoch": 0.6133333333333333, + "eps": 0, + "loss/policy_avg": 0.08223091810941696, + "loss/value_avg": 0.4521709978580475, + "lr": 3.5773620668448384e-06, + "objective/entropy": -57.002418518066406, + "objective/kl": 34.86107635498047, + "objective/non_score_reward": -1.743053674697876, + "objective/rlhf_reward": 12.421009063720703, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009696290362626314, + "policy/clipfrac_avg": 0.005794525146484375, + "policy/entropy_avg": 0.1826324462890625, + "step": 759, + "val/clipfrac_avg": 0.03934604302048683, + "val/num_eos_tokens": 137, + "val/ratio": 0.9880379438400269, + "val/ratio_var": 4.820780668524094e-05 + }, + { + "episode": 12160, + "epoch": 0.6141414141414141, + "eps": 0, + "loss/policy_avg": 0.012903919443488121, + "loss/value_avg": 0.22128555178642273, + "lr": 3.564562124876344e-06, + "objective/entropy": -58.77183532714844, + "objective/kl": 40.15751647949219, + "objective/non_score_reward": -2.007875919342041, + "objective/rlhf_reward": 13.171812057495117, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009755170904099941, + "policy/clipfrac_avg": 0.0075699929147958755, + "policy/entropy_avg": 0.1793661117553711, + "step": 760, + "val/clipfrac_avg": 0.007695037871599197, + "val/num_eos_tokens": 194, + "val/ratio": 0.9879107475280762, + "val/ratio_var": 9.122795017901808e-05 + }, + { + "episode": 12176, + "epoch": 0.6149494949494949, + "eps": 0, + "loss/policy_avg": 0.23269692063331604, + "loss/value_avg": 0.6034168004989624, + "lr": 3.5517724269001853e-06, + "objective/entropy": -92.11294555664062, + "objective/kl": 36.40907287597656, + "objective/non_score_reward": -1.8204538822174072, + "objective/rlhf_reward": 13.468608856201172, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009949133964255452, + "policy/clipfrac_avg": 0.012031201273202896, + "policy/entropy_avg": 0.19497203826904297, + "step": 761, + "val/clipfrac_avg": 0.03142271935939789, + "val/num_eos_tokens": 210, + "val/ratio": 0.9871051907539368, + "val/ratio_var": 7.474326412193477e-05 + }, + { + "episode": 12192, + "epoch": 0.6157575757575757, + "eps": 0, + "loss/policy_avg": -0.014487015083432198, + "loss/value_avg": 1.0809375047683716, + "lr": 3.538993064189954e-06, + "objective/entropy": -42.83490753173828, + "objective/kl": 32.94031524658203, + "objective/non_score_reward": -1.6470158100128174, + "objective/rlhf_reward": 13.567828178405762, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0011591427028179169, + "policy/clipfrac_avg": 0.004484353121370077, + "policy/entropy_avg": 0.21469688415527344, + "step": 762, + "val/clipfrac_avg": 0.1017560362815857, + "val/num_eos_tokens": 453, + "val/ratio": 0.9864501953125, + "val/ratio_var": 8.947903552325442e-05 + }, + { + "episode": 12208, + "epoch": 0.6165656565656565, + "eps": 0, + "loss/policy_avg": -0.0410991832613945, + "loss/value_avg": 1.3014378547668457, + "lr": 3.526224127945479e-06, + "objective/entropy": -71.51974487304688, + "objective/kl": 31.28769302368164, + "objective/non_score_reward": -1.5643846988677979, + "objective/rlhf_reward": 15.216865539550781, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007811100222170353, + "policy/clipfrac_avg": 0.0053521376103162766, + "policy/entropy_avg": 0.16764509677886963, + "step": 763, + "val/clipfrac_avg": 0.059704042971134186, + "val/num_eos_tokens": 203, + "val/ratio": 0.9894566535949707, + "val/ratio_var": 6.841208232799545e-05 + }, + { + "episode": 12224, + "epoch": 0.6173737373737374, + "eps": 0, + "loss/policy_avg": 0.112126424908638, + "loss/value_avg": 0.725554347038269, + "lr": 3.5134657092921877e-06, + "objective/entropy": -99.9854736328125, + "objective/kl": 41.09782028198242, + "objective/non_score_reward": -2.0548908710479736, + "objective/rlhf_reward": 10.624795913696289, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009035035036504269, + "policy/clipfrac_avg": 0.010427422821521759, + "policy/entropy_avg": 0.17998504638671875, + "step": 764, + "val/clipfrac_avg": 0.12764863669872284, + "val/num_eos_tokens": 226, + "val/ratio": 0.988477349281311, + "val/ratio_var": 8.909796451916918e-05 + }, + { + "episode": 12240, + "epoch": 0.6181818181818182, + "eps": 0, + "loss/policy_avg": 0.37758830189704895, + "loss/value_avg": 0.7465800046920776, + "lr": 3.500717899280442e-06, + "objective/entropy": -135.23390197753906, + "objective/kl": 41.466583251953125, + "objective/non_score_reward": -2.073329210281372, + "objective/rlhf_reward": 12.94229507446289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009621024364605546, + "policy/clipfrac_avg": 0.00628144945949316, + "policy/entropy_avg": 0.17942047119140625, + "step": 765, + "val/clipfrac_avg": 0.10096652805805206, + "val/num_eos_tokens": 233, + "val/ratio": 0.9877568483352661, + "val/ratio_var": 7.067532715154812e-05 + }, + { + "episode": 12256, + "epoch": 0.618989898989899, + "eps": 0, + "loss/policy_avg": -0.10782650113105774, + "loss/value_avg": 0.4996359944343567, + "lr": 3.4879807888849026e-06, + "objective/entropy": -86.19502258300781, + "objective/kl": 39.61420440673828, + "objective/non_score_reward": -1.9807100296020508, + "objective/rlhf_reward": 13.706789016723633, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009516358841210604, + "policy/clipfrac_avg": 0.01008635200560093, + "policy/entropy_avg": 0.17633438110351562, + "step": 766, + "val/clipfrac_avg": 0.05575938522815704, + "val/num_eos_tokens": 426, + "val/ratio": 0.9884860515594482, + "val/ratio_var": 5.413329563452862e-05 + }, + { + "episode": 12272, + "epoch": 0.6197979797979798, + "eps": 0, + "loss/policy_avg": 0.0611935630440712, + "loss/value_avg": 0.40072089433670044, + "lr": 3.475254469003865e-06, + "objective/entropy": -78.26728057861328, + "objective/kl": 38.48719787597656, + "objective/non_score_reward": -1.9243600368499756, + "objective/rlhf_reward": 11.661577224731445, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009466722258366644, + "policy/clipfrac_avg": 0.0042708818800747395, + "policy/entropy_avg": 0.17498779296875, + "step": 767, + "val/clipfrac_avg": 0.1151452288031578, + "val/num_eos_tokens": 415, + "val/ratio": 0.9874659776687622, + "val/ratio_var": 7.443072536261752e-05 + }, + { + "episode": 12288, + "epoch": 0.6206060606060606, + "eps": 0, + "loss/policy_avg": -0.07818017899990082, + "loss/value_avg": 0.588293731212616, + "lr": 3.4625390304586224e-06, + "objective/entropy": -45.31693649291992, + "objective/kl": 41.06260681152344, + "objective/non_score_reward": -2.0531303882598877, + "objective/rlhf_reward": 13.860933303833008, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0011651598615571856, + "policy/clipfrac_avg": 0.0047182347625494, + "policy/entropy_avg": 0.2090606689453125, + "step": 768, + "val/clipfrac_avg": 0.16338574886322021, + "val/num_eos_tokens": 9, + "val/ratio": 0.986497700214386, + "val/ratio_var": 8.175932453013957e-05 + }, + { + "episode": 12304, + "epoch": 0.6214141414141414, + "eps": 0, + "loss/policy_avg": -0.3265213966369629, + "loss/value_avg": 0.5878744125366211, + "lr": 3.4498345639928167e-06, + "objective/entropy": -103.96510314941406, + "objective/kl": 35.794551849365234, + "objective/non_score_reward": -1.7897276878356934, + "objective/rlhf_reward": 13.507146835327148, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010084104724228382, + "policy/clipfrac_avg": 0.0070042358711361885, + "policy/entropy_avg": 0.19075870513916016, + "step": 769, + "val/clipfrac_avg": 0.12878292798995972, + "val/num_eos_tokens": 8, + "val/ratio": 0.9861502051353455, + "val/ratio_var": 8.973718649940565e-05 + }, + { + "episode": 12320, + "epoch": 0.6222222222222222, + "eps": 0, + "loss/policy_avg": -0.24864813685417175, + "loss/value_avg": 0.703667402267456, + "lr": 3.4371411602717785e-06, + "objective/entropy": -92.59934997558594, + "objective/kl": 31.87890625, + "objective/non_score_reward": -1.5939452648162842, + "objective/rlhf_reward": 14.429492950439453, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007836055592633784, + "policy/clipfrac_avg": 0.004061062820255756, + "policy/entropy_avg": 0.13613224029541016, + "step": 770, + "val/clipfrac_avg": 0.14356926083564758, + "val/num_eos_tokens": 22, + "val/ratio": 0.9909623265266418, + "val/ratio_var": 6.252773891901597e-05 + }, + { + "episode": 12336, + "epoch": 0.623030303030303, + "eps": 0, + "loss/policy_avg": -0.11106158792972565, + "loss/value_avg": 0.7719787359237671, + "lr": 3.424458909881897e-06, + "objective/entropy": -201.825927734375, + "objective/kl": 32.05082321166992, + "objective/non_score_reward": -1.602541208267212, + "objective/rlhf_reward": 13.842771530151367, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009224842651747167, + "policy/clipfrac_avg": 0.004090993665158749, + "policy/entropy_avg": 0.19452500343322754, + "step": 771, + "val/clipfrac_avg": 0.06381858140230179, + "val/num_eos_tokens": 137, + "val/ratio": 0.9873221516609192, + "val/ratio_var": 0.00010800129530252889 + }, + { + "episode": 12352, + "epoch": 0.6238383838383839, + "eps": 0, + "loss/policy_avg": -0.3147702217102051, + "loss/value_avg": 1.0198739767074585, + "lr": 3.411787903329963e-06, + "objective/entropy": -97.80130767822266, + "objective/kl": 28.94087791442871, + "objective/non_score_reward": -1.4470438957214355, + "objective/rlhf_reward": 13.572486877441406, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008233180269598961, + "policy/clipfrac_avg": 0.007523335050791502, + "policy/entropy_avg": 0.17452716827392578, + "step": 772, + "val/clipfrac_avg": 0.021688329055905342, + "val/num_eos_tokens": 116, + "val/ratio": 0.9885031580924988, + "val/ratio_var": 6.793635111534968e-05 + }, + { + "episode": 12368, + "epoch": 0.6246464646464647, + "eps": 0, + "loss/policy_avg": -0.019919998943805695, + "loss/value_avg": 0.31691867113113403, + "lr": 3.3991282310425276e-06, + "objective/entropy": -61.794166564941406, + "objective/kl": 33.001129150390625, + "objective/non_score_reward": -1.6500564813613892, + "objective/rlhf_reward": 12.943693161010742, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009417567634955049, + "policy/clipfrac_avg": 0.005282199941575527, + "policy/entropy_avg": 0.1732311248779297, + "step": 773, + "val/clipfrac_avg": 0.05137361213564873, + "val/num_eos_tokens": 13, + "val/ratio": 0.9889352321624756, + "val/ratio_var": 3.9570189983351156e-05 + }, + { + "episode": 12384, + "epoch": 0.6254545454545455, + "eps": 0, + "loss/policy_avg": 0.010051839053630829, + "loss/value_avg": 0.5414019227027893, + "lr": 3.386479983365251e-06, + "objective/entropy": -73.91504669189453, + "objective/kl": 29.43255615234375, + "objective/non_score_reward": -1.4716278314590454, + "objective/rlhf_reward": 13.293996810913086, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0010180522222071886, + "policy/clipfrac_avg": 0.0050889430567622185, + "policy/entropy_avg": 0.2153911590576172, + "step": 774, + "val/clipfrac_avg": 0.041913747787475586, + "val/num_eos_tokens": 214, + "val/ratio": 0.986315906047821, + "val/ratio_var": 8.102346328087151e-05 + }, + { + "episode": 12400, + "epoch": 0.6262626262626263, + "eps": 0, + "loss/policy_avg": -0.040474407374858856, + "loss/value_avg": 0.30427631735801697, + "lr": 3.3738432505622653e-06, + "objective/entropy": -75.78406524658203, + "objective/kl": 31.846160888671875, + "objective/non_score_reward": -1.5923081636428833, + "objective/rlhf_reward": 11.946754455566406, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009044273756444454, + "policy/clipfrac_avg": 0.005002772435545921, + "policy/entropy_avg": 0.18775272369384766, + "step": 775, + "val/clipfrac_avg": 0.04153668135404587, + "val/num_eos_tokens": 243, + "val/ratio": 0.9877529144287109, + "val/ratio_var": 5.462392800836824e-05 + }, + { + "episode": 12416, + "epoch": 0.6270707070707071, + "eps": 0, + "loss/policy_avg": -0.25257226824760437, + "loss/value_avg": 0.47636741399765015, + "lr": 3.3612181228155237e-06, + "objective/entropy": -97.84140014648438, + "objective/kl": 35.974266052246094, + "objective/non_score_reward": -1.798713207244873, + "objective/rlhf_reward": 13.685661315917969, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009363207500427961, + "policy/clipfrac_avg": 0.0035052683670073748, + "policy/entropy_avg": 0.18935298919677734, + "step": 776, + "val/clipfrac_avg": 0.07260937243700027, + "val/num_eos_tokens": 26, + "val/ratio": 0.9877586364746094, + "val/ratio_var": 0.00010292992374161258 + }, + { + "episode": 12432, + "epoch": 0.6278787878787879, + "eps": 0, + "loss/policy_avg": 0.41764986515045166, + "loss/value_avg": 0.8629061579704285, + "lr": 3.3486046902241663e-06, + "objective/entropy": -40.24072265625, + "objective/kl": 29.197486877441406, + "objective/non_score_reward": -1.4598743915557861, + "objective/rlhf_reward": 12.883874893188477, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010831760009750724, + "policy/clipfrac_avg": 0.02700810506939888, + "policy/entropy_avg": 0.16212844848632812, + "step": 777, + "val/clipfrac_avg": 0.08423963189125061, + "val/num_eos_tokens": 192, + "val/ratio": 0.9885735511779785, + "val/ratio_var": 8.724266081117094e-05 + }, + { + "episode": 12448, + "epoch": 0.6286868686868687, + "eps": 0, + "loss/policy_avg": 0.2242026925086975, + "loss/value_avg": 0.9421164989471436, + "lr": 3.336003042803861e-06, + "objective/entropy": -69.478271484375, + "objective/kl": 33.57006072998047, + "objective/non_score_reward": -1.6785030364990234, + "objective/rlhf_reward": 14.055871963500977, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008899248205125332, + "policy/clipfrac_avg": 0.006825867109000683, + "policy/entropy_avg": 0.14767837524414062, + "step": 778, + "val/clipfrac_avg": 0.1501787304878235, + "val/num_eos_tokens": 109, + "val/ratio": 0.9905418753623962, + "val/ratio_var": 4.142183388466947e-05 + }, + { + "episode": 12464, + "epoch": 0.6294949494949494, + "eps": 0, + "loss/policy_avg": 0.03614666312932968, + "loss/value_avg": 0.7387021780014038, + "lr": 3.3234132704861786e-06, + "objective/entropy": -182.4083251953125, + "objective/kl": 31.944690704345703, + "objective/non_score_reward": -1.5972344875335693, + "objective/rlhf_reward": 12.043390274047852, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.000871892087161541, + "policy/clipfrac_avg": 0.004613575059920549, + "policy/entropy_avg": 0.1733226776123047, + "step": 779, + "val/clipfrac_avg": 0.1070820540189743, + "val/num_eos_tokens": 28, + "val/ratio": 0.9880519509315491, + "val/ratio_var": 9.137512097368017e-05 + }, + { + "episode": 12480, + "epoch": 0.6303030303030303, + "eps": 0, + "loss/policy_avg": 0.491243839263916, + "loss/value_avg": 1.7449349164962769, + "lr": 3.310835463117942e-06, + "objective/entropy": -80.43102264404297, + "objective/kl": 22.97904396057129, + "objective/non_score_reward": -1.1489522457122803, + "objective/rlhf_reward": 13.28073501586914, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000753985601477325, + "policy/clipfrac_avg": 0.013304230757057667, + "policy/entropy_avg": 0.16235482692718506, + "step": 780, + "val/clipfrac_avg": 0.12828537821769714, + "val/num_eos_tokens": 261, + "val/ratio": 0.991666316986084, + "val/ratio_var": 4.57862151961308e-05 + }, + { + "episode": 12496, + "epoch": 0.6311111111111111, + "eps": 0, + "loss/policy_avg": -0.02106083557009697, + "loss/value_avg": 0.26465463638305664, + "lr": 3.2982697104605855e-06, + "objective/entropy": -96.58609008789062, + "objective/kl": 37.96983337402344, + "objective/non_score_reward": -1.898491621017456, + "objective/rlhf_reward": 12.742133140563965, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008919077808968723, + "policy/clipfrac_avg": 0.004138793330639601, + "policy/entropy_avg": 0.176055908203125, + "step": 781, + "val/clipfrac_avg": 0.022776879370212555, + "val/num_eos_tokens": 232, + "val/ratio": 0.9880313873291016, + "val/ratio_var": 5.30094766872935e-05 + }, + { + "episode": 12512, + "epoch": 0.6319191919191919, + "eps": 0, + "loss/policy_avg": -0.21595685184001923, + "loss/value_avg": 0.1668311059474945, + "lr": 3.2857161021895125e-06, + "objective/entropy": -113.97618103027344, + "objective/kl": 30.898021697998047, + "objective/non_score_reward": -1.5449011325836182, + "objective/rlhf_reward": 15.892599105834961, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0008122409926727414, + "policy/clipfrac_avg": 0.007180417887866497, + "policy/entropy_avg": 0.15228557586669922, + "step": 782, + "val/clipfrac_avg": 0.04710119590163231, + "val/num_eos_tokens": 88, + "val/ratio": 0.9905120134353638, + "val/ratio_var": 4.6862616727594286e-05 + }, + { + "episode": 12528, + "epoch": 0.6327272727272727, + "eps": 0, + "loss/policy_avg": -0.02830284647643566, + "loss/value_avg": 0.470698744058609, + "lr": 3.273174727893463e-06, + "objective/entropy": -119.84375762939453, + "objective/kl": 35.46149444580078, + "objective/non_score_reward": -1.773074746131897, + "objective/rlhf_reward": 13.00036334991455, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000846280949190259, + "policy/clipfrac_avg": 0.006307992152869701, + "policy/entropy_avg": 0.15985870361328125, + "step": 783, + "val/clipfrac_avg": 0.09303553402423859, + "val/num_eos_tokens": 242, + "val/ratio": 0.9886760711669922, + "val/ratio_var": 8.341170905623585e-05 + }, + { + "episode": 12544, + "epoch": 0.6335353535353535, + "eps": 0, + "loss/policy_avg": -0.03483296558260918, + "loss/value_avg": 0.2278536707162857, + "lr": 3.2606456770738636e-06, + "objective/entropy": -32.83402633666992, + "objective/kl": 35.053001403808594, + "objective/non_score_reward": -1.7526500225067139, + "objective/rlhf_reward": 13.708287239074707, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010523913661018014, + "policy/clipfrac_avg": 0.005916148889809847, + "policy/entropy_avg": 0.2171039581298828, + "step": 784, + "val/clipfrac_avg": 0.007893375121057034, + "val/num_eos_tokens": 463, + "val/ratio": 0.9859672784805298, + "val/ratio_var": 5.139111090102233e-05 + }, + { + "episode": 12560, + "epoch": 0.6343434343434343, + "eps": 0, + "loss/policy_avg": 0.17171788215637207, + "loss/value_avg": 0.5937256217002869, + "lr": 3.2481290391442034e-06, + "objective/entropy": -109.50154113769531, + "objective/kl": 41.38114929199219, + "objective/non_score_reward": -2.0690574645996094, + "objective/rlhf_reward": 13.82156753540039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009320294484496117, + "policy/clipfrac_avg": 0.005337849725037813, + "policy/entropy_avg": 0.1828467845916748, + "step": 785, + "val/clipfrac_avg": 0.0434221550822258, + "val/num_eos_tokens": 252, + "val/ratio": 0.9880971908569336, + "val/ratio_var": 7.803420157870278e-05 + }, + { + "episode": 12576, + "epoch": 0.6351515151515151, + "eps": 0, + "loss/policy_avg": -0.12326152622699738, + "loss/value_avg": 0.22453081607818604, + "lr": 3.235624903429374e-06, + "objective/entropy": -80.62982940673828, + "objective/kl": 39.33965301513672, + "objective/non_score_reward": -1.9669828414916992, + "objective/rlhf_reward": 14.3767671585083, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009726607240736485, + "policy/clipfrac_avg": 0.005861825309693813, + "policy/entropy_avg": 0.18674850463867188, + "step": 786, + "val/clipfrac_avg": 0.04197417199611664, + "val/num_eos_tokens": 55, + "val/ratio": 0.9867269396781921, + "val/ratio_var": 3.261497477069497e-05 + }, + { + "episode": 12592, + "epoch": 0.635959595959596, + "eps": 0, + "loss/policy_avg": 0.02891068160533905, + "loss/value_avg": 0.2414318025112152, + "lr": 3.2231333591650567e-06, + "objective/entropy": -64.31414031982422, + "objective/kl": 46.58558654785156, + "objective/non_score_reward": -2.3292794227600098, + "objective/rlhf_reward": 14.576970100402832, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009546708315610886, + "policy/clipfrac_avg": 0.00552853848785162, + "policy/entropy_avg": 0.189300537109375, + "step": 787, + "val/clipfrac_avg": 0.0013757861452177167, + "val/num_eos_tokens": 363, + "val/ratio": 0.987205982208252, + "val/ratio_var": 3.5099023079965264e-05 + }, + { + "episode": 12608, + "epoch": 0.6367676767676768, + "eps": 0, + "loss/policy_avg": -0.19538097083568573, + "loss/value_avg": 0.7160595655441284, + "lr": 3.210654495497068e-06, + "objective/entropy": -57.81846237182617, + "objective/kl": 42.85956573486328, + "objective/non_score_reward": -2.1429784297943115, + "objective/rlhf_reward": 12.14999008178711, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0011135776294395328, + "policy/clipfrac_avg": 0.009378955699503422, + "policy/entropy_avg": 0.2034454345703125, + "step": 788, + "val/clipfrac_avg": 0.1690627932548523, + "val/num_eos_tokens": 481, + "val/ratio": 0.9861621856689453, + "val/ratio_var": 9.24188134376891e-05 + }, + { + "episode": 12624, + "epoch": 0.6375757575757576, + "eps": 0, + "loss/policy_avg": 0.048018310219049454, + "loss/value_avg": 0.4312187135219574, + "lr": 3.198188401480734e-06, + "objective/entropy": -76.67124938964844, + "objective/kl": 36.849369049072266, + "objective/non_score_reward": -1.8424686193466187, + "objective/rlhf_reward": 12.25128173828125, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009952555410563946, + "policy/clipfrac_avg": 0.004150367341935635, + "policy/entropy_avg": 0.18071389198303223, + "step": 789, + "val/clipfrac_avg": 0.08804323524236679, + "val/num_eos_tokens": 357, + "val/ratio": 0.9867403507232666, + "val/ratio_var": 0.00013288675108924508 + }, + { + "episode": 12640, + "epoch": 0.6383838383838384, + "eps": 0, + "loss/policy_avg": 0.18971526622772217, + "loss/value_avg": 0.6626036167144775, + "lr": 3.185735166080244e-06, + "objective/entropy": -126.28926086425781, + "objective/kl": 33.96638488769531, + "objective/non_score_reward": -1.6983189582824707, + "objective/rlhf_reward": 12.973556518554688, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0007302669691853225, + "policy/clipfrac_avg": 0.008397979661822319, + "policy/entropy_avg": 0.14786386489868164, + "step": 790, + "val/clipfrac_avg": 0.06916166841983795, + "val/num_eos_tokens": 600, + "val/ratio": 0.9910507798194885, + "val/ratio_var": 5.283964492264204e-05 + }, + { + "episode": 12656, + "epoch": 0.6391919191919192, + "eps": 0, + "loss/policy_avg": -0.4279332160949707, + "loss/value_avg": 0.5117957592010498, + "lr": 3.173294878168025e-06, + "objective/entropy": -75.64164733886719, + "objective/kl": 40.16947555541992, + "objective/non_score_reward": -2.0084738731384277, + "objective/rlhf_reward": 12.86652660369873, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008897786610759795, + "policy/clipfrac_avg": 0.004540015943348408, + "policy/entropy_avg": 0.18616324663162231, + "step": 791, + "val/clipfrac_avg": 0.0338541679084301, + "val/num_eos_tokens": 224, + "val/ratio": 0.9873502254486084, + "val/ratio_var": 6.438903801608831e-05 + }, + { + "episode": 12672, + "epoch": 0.64, + "eps": 0, + "loss/policy_avg": -0.30223172903060913, + "loss/value_avg": 0.7414761185646057, + "lr": 3.160867626524107e-06, + "objective/entropy": -36.1086540222168, + "objective/kl": 35.08636474609375, + "objective/non_score_reward": -1.7543182373046875, + "objective/rlhf_reward": 14.995681762695312, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010164354462176561, + "policy/clipfrac_avg": 0.0033426620066165924, + "policy/entropy_avg": 0.20384597778320312, + "step": 792, + "val/clipfrac_avg": 0.08375494927167892, + "val/num_eos_tokens": 121, + "val/ratio": 0.9862762093544006, + "val/ratio_var": 8.216432615881786e-05 + }, + { + "episode": 12688, + "epoch": 0.6408080808080808, + "eps": 0, + "loss/policy_avg": 0.11556120216846466, + "loss/value_avg": 0.44696569442749023, + "lr": 3.1484534998354832e-06, + "objective/entropy": -52.89218521118164, + "objective/kl": 33.01978302001953, + "objective/non_score_reward": -1.650989294052124, + "objective/rlhf_reward": 12.270885467529297, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009637182811275125, + "policy/clipfrac_avg": 0.006595691200345755, + "policy/entropy_avg": 0.18551349639892578, + "step": 793, + "val/clipfrac_avg": 0.0313958078622818, + "val/num_eos_tokens": 483, + "val/ratio": 0.987555742263794, + "val/ratio_var": 4.9779242544900626e-05 + }, + { + "episode": 12704, + "epoch": 0.6416161616161616, + "eps": 0, + "loss/policy_avg": -0.11985203623771667, + "loss/value_avg": 0.5595318078994751, + "lr": 3.136052586695481e-06, + "objective/entropy": -44.2855110168457, + "objective/kl": 38.74113845825195, + "objective/non_score_reward": -1.9370570182800293, + "objective/rlhf_reward": 13.531692504882812, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009892593370750546, + "policy/clipfrac_avg": 0.006002710200846195, + "policy/entropy_avg": 0.20067405700683594, + "step": 794, + "val/clipfrac_avg": 0.051367055624723434, + "val/num_eos_tokens": 146, + "val/ratio": 0.98707115650177, + "val/ratio_var": 6.210069841472432e-05 + }, + { + "episode": 12720, + "epoch": 0.6424242424242425, + "eps": 0, + "loss/policy_avg": 0.06161116436123848, + "loss/value_avg": 0.20387178659439087, + "lr": 3.12366497560313e-06, + "objective/entropy": -47.063812255859375, + "objective/kl": 36.60810852050781, + "objective/non_score_reward": -1.8304054737091064, + "objective/rlhf_reward": 13.661782264709473, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000998336123302579, + "policy/clipfrac_avg": 0.004183639772236347, + "policy/entropy_avg": 0.19140625, + "step": 795, + "val/clipfrac_avg": 0.08834637701511383, + "val/num_eos_tokens": 342, + "val/ratio": 0.9874628782272339, + "val/ratio_var": 3.3792719477787614e-05 + }, + { + "episode": 12736, + "epoch": 0.6432323232323233, + "eps": 0, + "loss/policy_avg": 0.02652081847190857, + "loss/value_avg": 0.4432865381240845, + "lr": 3.1112907549625304e-06, + "objective/entropy": -104.79203796386719, + "objective/kl": 35.12114715576172, + "objective/non_score_reward": -1.7560575008392334, + "objective/rlhf_reward": 13.486129760742188, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008146626059897244, + "policy/clipfrac_avg": 0.0056906514801084995, + "policy/entropy_avg": 0.1502237319946289, + "step": 796, + "val/clipfrac_avg": 0.061473242938518524, + "val/num_eos_tokens": 748, + "val/ratio": 0.9898946285247803, + "val/ratio_var": 4.619114042725414e-05 + }, + { + "episode": 12752, + "epoch": 0.6440404040404041, + "eps": 0, + "loss/policy_avg": 0.06834741681814194, + "loss/value_avg": 0.5731651782989502, + "lr": 3.098930013082223e-06, + "objective/entropy": -88.31727600097656, + "objective/kl": 38.224117279052734, + "objective/non_score_reward": -1.9112060070037842, + "objective/rlhf_reward": 14.698169708251953, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008200570591725409, + "policy/clipfrac_avg": 0.007693252060562372, + "policy/entropy_avg": 0.1607208251953125, + "step": 797, + "val/clipfrac_avg": 0.11237858235836029, + "val/num_eos_tokens": 240, + "val/ratio": 0.9905504584312439, + "val/ratio_var": 4.706854815594852e-05 + }, + { + "episode": 12768, + "epoch": 0.6448484848484849, + "eps": 0, + "loss/policy_avg": -0.09761235117912292, + "loss/value_avg": 0.7088909149169922, + "lr": 3.0865828381745515e-06, + "objective/entropy": -124.7913589477539, + "objective/kl": 29.702392578125, + "objective/non_score_reward": -1.4851195812225342, + "objective/rlhf_reward": 13.68675422668457, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008782441727817059, + "policy/clipfrac_avg": 0.00598434591665864, + "policy/entropy_avg": 0.16924285888671875, + "step": 798, + "val/clipfrac_avg": 0.030641667544841766, + "val/num_eos_tokens": 454, + "val/ratio": 0.9882635474205017, + "val/ratio_var": 7.515518518630415e-05 + }, + { + "episode": 12784, + "epoch": 0.6456565656565657, + "eps": 0, + "loss/policy_avg": 0.14146611094474792, + "loss/value_avg": 0.7321012616157532, + "lr": 3.074249318355046e-06, + "objective/entropy": -101.5077133178711, + "objective/kl": 25.74894142150879, + "objective/non_score_reward": -1.2874470949172974, + "objective/rlhf_reward": 13.735990524291992, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007512210868299007, + "policy/clipfrac_avg": 0.011982694268226624, + "policy/entropy_avg": 0.15892601013183594, + "step": 799, + "val/clipfrac_avg": 0.012677992694079876, + "val/num_eos_tokens": 12, + "val/ratio": 0.9910134077072144, + "val/ratio_var": 6.09022390563041e-05 + }, + { + "episode": 12800, + "epoch": 0.6464646464646465, + "eps": 0, + "loss/policy_avg": -0.042851828038692474, + "loss/value_avg": 0.3452759385108948, + "lr": 3.0619295416417856e-06, + "objective/entropy": -68.88214874267578, + "objective/kl": 34.023399353027344, + "objective/non_score_reward": -1.7011699676513672, + "objective/rlhf_reward": 12.494142532348633, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010239251423627138, + "policy/clipfrac_avg": 0.010894561186432838, + "policy/entropy_avg": 0.17738628387451172, + "step": 800, + "val/clipfrac_avg": 0.026722928509116173, + "val/num_eos_tokens": 67, + "val/ratio": 0.9888595342636108, + "val/ratio_var": 6.888628558954224e-05 + }, + { + "episode": 12816, + "epoch": 0.6472727272727272, + "eps": 0, + "loss/policy_avg": -0.05754968896508217, + "loss/value_avg": 0.402332067489624, + "lr": 3.049623595954766e-06, + "objective/entropy": -89.62500762939453, + "objective/kl": 32.48891067504883, + "objective/non_score_reward": -1.6244455575942993, + "objective/rlhf_reward": 14.227116584777832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009385538287460804, + "policy/clipfrac_avg": 0.004920909181237221, + "policy/entropy_avg": 0.17440450191497803, + "step": 801, + "val/clipfrac_avg": 0.03176930546760559, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880223274230957, + "val/ratio_var": 5.388163845054805e-05 + }, + { + "episode": 12832, + "epoch": 0.648080808080808, + "eps": 0, + "loss/policy_avg": 0.2483101636171341, + "loss/value_avg": 0.23584425449371338, + "lr": 3.0373315691152856e-06, + "objective/entropy": -89.31193542480469, + "objective/kl": 40.699127197265625, + "objective/non_score_reward": -2.034956455230713, + "objective/rlhf_reward": 12.160355567932129, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0008679171442054212, + "policy/clipfrac_avg": 0.004822667222470045, + "policy/entropy_avg": 0.1660480499267578, + "step": 802, + "val/clipfrac_avg": 0.05813268944621086, + "val/num_eos_tokens": 149, + "val/ratio": 0.9890165328979492, + "val/ratio_var": 5.875104761798866e-05 + }, + { + "episode": 12848, + "epoch": 0.6488888888888888, + "eps": 0, + "loss/policy_avg": -0.10593897104263306, + "loss/value_avg": 0.5456434488296509, + "lr": 3.0250535488453077e-06, + "objective/entropy": -148.4324188232422, + "objective/kl": 38.51166915893555, + "objective/non_score_reward": -1.9255836009979248, + "objective/rlhf_reward": 11.918167114257812, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0007962615345604718, + "policy/clipfrac_avg": 0.0040261996909976006, + "policy/entropy_avg": 0.16094493865966797, + "step": 803, + "val/clipfrac_avg": 0.04293014109134674, + "val/num_eos_tokens": 331, + "val/ratio": 0.9893560409545898, + "val/ratio_var": 6.857667176518589e-05 + }, + { + "episode": 12864, + "epoch": 0.6496969696969697, + "eps": 0, + "loss/policy_avg": 0.06602153182029724, + "loss/value_avg": 0.2823004722595215, + "lr": 3.0127896227668367e-06, + "objective/entropy": -115.89801788330078, + "objective/kl": 40.921504974365234, + "objective/non_score_reward": -2.0460751056671143, + "objective/rlhf_reward": 12.867987632751465, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008664772612974048, + "policy/clipfrac_avg": 0.00966357346624136, + "policy/entropy_avg": 0.151214599609375, + "step": 804, + "val/clipfrac_avg": 0.03888742998242378, + "val/num_eos_tokens": 224, + "val/ratio": 0.9900601506233215, + "val/ratio_var": 5.00909227412194e-05 + }, + { + "episode": 12880, + "epoch": 0.6505050505050505, + "eps": 0, + "loss/policy_avg": -0.39892637729644775, + "loss/value_avg": 0.4240272641181946, + "lr": 3.000539878401296e-06, + "objective/entropy": -77.75199890136719, + "objective/kl": 35.056907653808594, + "objective/non_score_reward": -1.7528454065322876, + "objective/rlhf_reward": 14.00496768951416, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.001023960066959262, + "policy/clipfrac_avg": 0.004629913717508316, + "policy/entropy_avg": 0.20476627349853516, + "step": 805, + "val/clipfrac_avg": 0.07910472899675369, + "val/num_eos_tokens": 219, + "val/ratio": 0.9861233830451965, + "val/ratio_var": 8.373144373763353e-05 + }, + { + "episode": 12896, + "epoch": 0.6513131313131313, + "eps": 0, + "loss/policy_avg": -0.23323668539524078, + "loss/value_avg": 0.4607469439506531, + "lr": 2.9883044031688995e-06, + "objective/entropy": -85.76292419433594, + "objective/kl": 40.88092803955078, + "objective/non_score_reward": -2.044046401977539, + "objective/rlhf_reward": 14.174703598022461, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010081828804686666, + "policy/clipfrac_avg": 0.0041284579783678055, + "policy/entropy_avg": 0.1921253204345703, + "step": 806, + "val/clipfrac_avg": 0.16490784287452698, + "val/num_eos_tokens": 249, + "val/ratio": 0.9875838756561279, + "val/ratio_var": 6.450649379985407e-05 + }, + { + "episode": 12912, + "epoch": 0.6521212121212121, + "eps": 0, + "loss/policy_avg": -0.11845395714044571, + "loss/value_avg": 0.4592535197734833, + "lr": 2.976083284388031e-06, + "objective/entropy": -62.67930221557617, + "objective/kl": 33.47719955444336, + "objective/non_score_reward": -1.6738600730895996, + "objective/rlhf_reward": 12.771451950073242, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011091704946011305, + "policy/clipfrac_avg": 0.00747725460678339, + "policy/entropy_avg": 0.1983814239501953, + "step": 807, + "val/clipfrac_avg": 0.033114466816186905, + "val/num_eos_tokens": 17, + "val/ratio": 0.9877710938453674, + "val/ratio_var": 6.532426050398499e-05 + }, + { + "episode": 12928, + "epoch": 0.6529292929292929, + "eps": 0, + "loss/policy_avg": 0.0469900444149971, + "loss/value_avg": 0.26954370737075806, + "lr": 2.9638766092746207e-06, + "objective/entropy": -88.9464111328125, + "objective/kl": 34.25027847290039, + "objective/non_score_reward": -1.7125139236450195, + "objective/rlhf_reward": 12.56873607635498, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009448326891288161, + "policy/clipfrac_avg": 0.006387755274772644, + "policy/entropy_avg": 0.18566322326660156, + "step": 808, + "val/clipfrac_avg": 0.018177732825279236, + "val/num_eos_tokens": 761, + "val/ratio": 0.9882063269615173, + "val/ratio_var": 5.754105222877115e-05 + }, + { + "episode": 12944, + "epoch": 0.6537373737373737, + "eps": 0, + "loss/policy_avg": 0.00823296420276165, + "loss/value_avg": 0.44794195890426636, + "lr": 2.9516844649415165e-06, + "objective/entropy": -93.69944763183594, + "objective/kl": 40.86418151855469, + "objective/non_score_reward": -2.0432090759277344, + "objective/rlhf_reward": 13.542728424072266, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000799038156401366, + "policy/clipfrac_avg": 0.006408605724573135, + "policy/entropy_avg": 0.14969253540039062, + "step": 809, + "val/clipfrac_avg": 0.08976449072360992, + "val/num_eos_tokens": 215, + "val/ratio": 0.989780068397522, + "val/ratio_var": 6.0727201343979686e-05 + }, + { + "episode": 12960, + "epoch": 0.6545454545454545, + "eps": 0, + "loss/policy_avg": 0.024010583758354187, + "loss/value_avg": 0.3378852605819702, + "lr": 2.9395069383978725e-06, + "objective/entropy": -41.68758773803711, + "objective/kl": 35.73942184448242, + "objective/non_score_reward": -1.7869713306427002, + "objective/rlhf_reward": 15.314591407775879, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008846010314300656, + "policy/clipfrac_avg": 0.00692939106374979, + "policy/entropy_avg": 0.17317676544189453, + "step": 810, + "val/clipfrac_avg": 0.0032868448179215193, + "val/num_eos_tokens": 327, + "val/ratio": 0.9892865419387817, + "val/ratio_var": 4.3601932702586055e-05 + }, + { + "episode": 12976, + "epoch": 0.6553535353535354, + "eps": 0, + "loss/policy_avg": 0.1456170380115509, + "loss/value_avg": 0.5115708112716675, + "lr": 2.9273441165485227e-06, + "objective/entropy": -45.97090148925781, + "objective/kl": 36.01569366455078, + "objective/non_score_reward": -1.8007844686508179, + "objective/rlhf_reward": 14.23046588897705, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009762131376191974, + "policy/clipfrac_avg": 0.004803863354027271, + "policy/entropy_avg": 0.18830299377441406, + "step": 811, + "val/clipfrac_avg": 0.07713419198989868, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878898859024048, + "val/ratio_var": 4.7962184908101335e-05 + }, + { + "episode": 12992, + "epoch": 0.6561616161616162, + "eps": 0, + "loss/policy_avg": 0.1002146378159523, + "loss/value_avg": 0.558029294013977, + "lr": 2.9151960861933616e-06, + "objective/entropy": -66.14154052734375, + "objective/kl": 36.481346130371094, + "objective/non_score_reward": -1.8240673542022705, + "objective/rlhf_reward": 14.175932884216309, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009823704604059458, + "policy/clipfrac_avg": 0.0034461808390915394, + "policy/entropy_avg": 0.1955432891845703, + "step": 812, + "val/clipfrac_avg": 0.10748951137065887, + "val/num_eos_tokens": 292, + "val/ratio": 0.986059308052063, + "val/ratio_var": 5.328569022822194e-05 + }, + { + "episode": 13008, + "epoch": 0.656969696969697, + "eps": 0, + "loss/policy_avg": -0.10509809851646423, + "loss/value_avg": 0.46006062626838684, + "lr": 2.9030629340267165e-06, + "objective/entropy": -110.846923828125, + "objective/kl": 36.021034240722656, + "objective/non_score_reward": -1.8010516166687012, + "objective/rlhf_reward": 16.33176040649414, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008035163627937436, + "policy/clipfrac_avg": 0.005940265022218227, + "policy/entropy_avg": 0.15110206604003906, + "step": 813, + "val/clipfrac_avg": 0.07604548335075378, + "val/num_eos_tokens": 523, + "val/ratio": 0.9898133277893066, + "val/ratio_var": 6.0903592384420335e-05 + }, + { + "episode": 13024, + "epoch": 0.6577777777777778, + "eps": 0, + "loss/policy_avg": -0.11711955070495605, + "loss/value_avg": 1.2142528295516968, + "lr": 2.890944746636748e-06, + "objective/entropy": -40.75424575805664, + "objective/kl": 34.03726577758789, + "objective/non_score_reward": -1.701863408088684, + "objective/rlhf_reward": 13.091104507446289, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.000980376498773694, + "policy/clipfrac_avg": 0.008426186628639698, + "policy/entropy_avg": 0.20235157012939453, + "step": 814, + "val/clipfrac_avg": 0.02976190485060215, + "val/num_eos_tokens": 113, + "val/ratio": 0.9871314764022827, + "val/ratio_var": 6.27562912995927e-05 + }, + { + "episode": 13040, + "epoch": 0.6585858585858586, + "eps": 0, + "loss/policy_avg": -0.18143659830093384, + "loss/value_avg": 0.571576714515686, + "lr": 2.8788416105048124e-06, + "objective/entropy": -141.41366577148438, + "objective/kl": 42.53788757324219, + "objective/non_score_reward": -2.126894474029541, + "objective/rlhf_reward": 13.537168502807617, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008768361294642091, + "policy/clipfrac_avg": 0.006732264067977667, + "policy/entropy_avg": 0.17724037170410156, + "step": 815, + "val/clipfrac_avg": 0.03276197984814644, + "val/num_eos_tokens": 221, + "val/ratio": 0.9885480403900146, + "val/ratio_var": 8.229191007558256e-05 + }, + { + "episode": 13056, + "epoch": 0.6593939393939394, + "eps": 0, + "loss/policy_avg": 0.3017633855342865, + "loss/value_avg": 0.32543864846229553, + "lr": 2.8667536120048616e-06, + "objective/entropy": -61.10264587402344, + "objective/kl": 35.6143684387207, + "objective/non_score_reward": -1.7807183265686035, + "objective/rlhf_reward": 14.391157150268555, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009194887243211269, + "policy/clipfrac_avg": 0.018624408170580864, + "policy/entropy_avg": 0.14903783798217773, + "step": 816, + "val/clipfrac_avg": 0.0073128631338477135, + "val/num_eos_tokens": 112, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.833535033161752e-05 + }, + { + "episode": 13072, + "epoch": 0.6602020202020202, + "eps": 0, + "loss/policy_avg": 0.06899265199899673, + "loss/value_avg": 0.6989455223083496, + "lr": 2.854680837402804e-06, + "objective/entropy": -117.86632537841797, + "objective/kl": 34.15174102783203, + "objective/non_score_reward": -1.7075870037078857, + "objective/rlhf_reward": 14.511162757873535, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009505614871159196, + "policy/clipfrac_avg": 0.008451011963188648, + "policy/entropy_avg": 0.1699972152709961, + "step": 817, + "val/clipfrac_avg": 0.000589622650295496, + "val/num_eos_tokens": 435, + "val/ratio": 0.9876314401626587, + "val/ratio_var": 9.73327478277497e-05 + }, + { + "episode": 13088, + "epoch": 0.661010101010101, + "eps": 0, + "loss/policy_avg": -0.05428570508956909, + "loss/value_avg": 0.2928343415260315, + "lr": 2.8426233728559172e-06, + "objective/entropy": -52.28473663330078, + "objective/kl": 40.512550354003906, + "objective/non_score_reward": -2.025627613067627, + "objective/rlhf_reward": 14.458747863769531, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008845083648338914, + "policy/clipfrac_avg": 0.005094105843454599, + "policy/entropy_avg": 0.16685867309570312, + "step": 818, + "val/clipfrac_avg": 0.09271319210529327, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885064959526062, + "val/ratio_var": 4.853576319874264e-05 + }, + { + "episode": 13104, + "epoch": 0.6618181818181819, + "eps": 0, + "loss/policy_avg": 0.12085426598787308, + "loss/value_avg": 0.5670195817947388, + "lr": 2.83058130441221e-06, + "objective/entropy": -196.3238983154297, + "objective/kl": 42.012672424316406, + "objective/non_score_reward": -2.1006336212158203, + "objective/rlhf_reward": 12.51655387878418, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007904545636847615, + "policy/clipfrac_avg": 0.006419865880161524, + "policy/entropy_avg": 0.1437664031982422, + "step": 819, + "val/clipfrac_avg": 0.05388574302196503, + "val/num_eos_tokens": 496, + "val/ratio": 0.9899193644523621, + "val/ratio_var": 4.292546145734377e-05 + }, + { + "episode": 13120, + "epoch": 0.6626262626262627, + "eps": 0, + "loss/policy_avg": 0.6058844327926636, + "loss/value_avg": 0.7294787168502808, + "lr": 2.8185547180098195e-06, + "objective/entropy": -58.513553619384766, + "objective/kl": 38.73785400390625, + "objective/non_score_reward": -1.9368927478790283, + "objective/rlhf_reward": 12.883419036865234, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010288474150002003, + "policy/clipfrac_avg": 0.021498143672943115, + "policy/entropy_avg": 0.20896148681640625, + "step": 820, + "val/clipfrac_avg": 0.0723545029759407, + "val/num_eos_tokens": 134, + "val/ratio": 0.9887969493865967, + "val/ratio_var": 4.8135509132407606e-05 + }, + { + "episode": 13136, + "epoch": 0.6634343434343435, + "eps": 0, + "loss/policy_avg": -0.21932660043239594, + "loss/value_avg": 0.45023149251937866, + "lr": 2.8065436994763963e-06, + "objective/entropy": -38.43750762939453, + "objective/kl": 33.36050033569336, + "objective/non_score_reward": -1.6680251359939575, + "objective/rlhf_reward": 13.542912483215332, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010698122205212712, + "policy/clipfrac_avg": 0.004931025672703981, + "policy/entropy_avg": 0.19058465957641602, + "step": 821, + "val/clipfrac_avg": 0.00852490495890379, + "val/num_eos_tokens": 190, + "val/ratio": 0.9864101409912109, + "val/ratio_var": 5.521172352018766e-05 + }, + { + "episode": 13152, + "epoch": 0.6642424242424242, + "eps": 0, + "loss/policy_avg": -0.01703064516186714, + "loss/value_avg": 0.2902008295059204, + "lr": 2.794548334528486e-06, + "objective/entropy": 13.10763931274414, + "objective/kl": 44.57916259765625, + "objective/non_score_reward": -2.2289581298828125, + "objective/rlhf_reward": 13.364792823791504, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0014038155786693096, + "policy/clipfrac_avg": 0.006509849801659584, + "policy/entropy_avg": 0.26366424560546875, + "step": 822, + "val/clipfrac_avg": 0.03128276392817497, + "val/num_eos_tokens": 35, + "val/ratio": 0.9824488162994385, + "val/ratio_var": 4.7705398174002767e-05 + }, + { + "episode": 13168, + "epoch": 0.665050505050505, + "eps": 0, + "loss/policy_avg": -0.02819274365901947, + "loss/value_avg": 0.2861873507499695, + "lr": 2.782568708770933e-06, + "objective/entropy": -94.84840393066406, + "objective/kl": 42.75294494628906, + "objective/non_score_reward": -2.1376476287841797, + "objective/rlhf_reward": 12.26860237121582, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010491260327398777, + "policy/clipfrac_avg": 0.004985281266272068, + "policy/entropy_avg": 0.2000885009765625, + "step": 823, + "val/clipfrac_avg": 0.00754884909838438, + "val/num_eos_tokens": 289, + "val/ratio": 0.9866741895675659, + "val/ratio_var": 4.918476770399138e-05 + }, + { + "episode": 13184, + "epoch": 0.6658585858585858, + "eps": 0, + "loss/policy_avg": -0.024818487465381622, + "loss/value_avg": 0.3611629605293274, + "lr": 2.7706049076962483e-06, + "objective/entropy": -104.7535400390625, + "objective/kl": 45.08838653564453, + "objective/non_score_reward": -2.2544193267822266, + "objective/rlhf_reward": 12.214330673217773, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009155337465927005, + "policy/clipfrac_avg": 0.008831838145852089, + "policy/entropy_avg": 0.16748523712158203, + "step": 824, + "val/clipfrac_avg": 0.0024258445482701063, + "val/num_eos_tokens": 163, + "val/ratio": 0.9890375137329102, + "val/ratio_var": 5.131318175699562e-05 + }, + { + "episode": 13200, + "epoch": 0.6666666666666666, + "eps": 0, + "loss/policy_avg": -0.28224268555641174, + "loss/value_avg": 0.8720104694366455, + "lr": 2.7586570166840154e-06, + "objective/entropy": -74.55836486816406, + "objective/kl": 30.59856605529785, + "objective/non_score_reward": -1.52992844581604, + "objective/rlhf_reward": 15.329446792602539, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008330290438607335, + "policy/clipfrac_avg": 0.010464916005730629, + "policy/entropy_avg": 0.15100550651550293, + "step": 825, + "val/clipfrac_avg": 0.08580771088600159, + "val/num_eos_tokens": 460, + "val/ratio": 0.9896235466003418, + "val/ratio_var": 9.328931628260761e-05 + }, + { + "episode": 13216, + "epoch": 0.6674747474747474, + "eps": 0, + "loss/policy_avg": -0.1715128868818283, + "loss/value_avg": 0.4540337920188904, + "lr": 2.746725121000273e-06, + "objective/entropy": -43.89616394042969, + "objective/kl": 41.760223388671875, + "objective/non_score_reward": -2.0880112648010254, + "objective/rlhf_reward": 13.872926712036133, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010317859705537558, + "policy/clipfrac_avg": 0.0054151020012795925, + "policy/entropy_avg": 0.19500446319580078, + "step": 826, + "val/clipfrac_avg": 0.03631855547428131, + "val/num_eos_tokens": 334, + "val/ratio": 0.9874616265296936, + "val/ratio_var": 6.214805034687743e-05 + }, + { + "episode": 13232, + "epoch": 0.6682828282828283, + "eps": 0, + "loss/policy_avg": -0.077702596783638, + "loss/value_avg": 0.6576616168022156, + "lr": 2.734809305796915e-06, + "objective/entropy": -13.071683883666992, + "objective/kl": 34.05264663696289, + "objective/non_score_reward": -1.7026324272155762, + "objective/rlhf_reward": 14.656742095947266, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0012533586705103517, + "policy/clipfrac_avg": 0.00517519935965538, + "policy/entropy_avg": 0.21735763549804688, + "step": 827, + "val/clipfrac_avg": 0.09026823937892914, + "val/num_eos_tokens": 468, + "val/ratio": 0.9863014221191406, + "val/ratio_var": 0.00010488701082067564 + }, + { + "episode": 13248, + "epoch": 0.6690909090909091, + "eps": 0, + "loss/policy_avg": -0.11604561656713486, + "loss/value_avg": 0.18491807579994202, + "lr": 2.7229096561110703e-06, + "objective/entropy": -132.30560302734375, + "objective/kl": 37.0118522644043, + "objective/non_score_reward": -1.8505926132202148, + "objective/rlhf_reward": 15.282218933105469, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008336640894412994, + "policy/clipfrac_avg": 0.0030343476682901382, + "policy/entropy_avg": 0.16221046447753906, + "step": 828, + "val/clipfrac_avg": 0.07020965218544006, + "val/num_eos_tokens": 41, + "val/ratio": 0.9894965291023254, + "val/ratio_var": 4.505051037995145e-05 + }, + { + "episode": 13264, + "epoch": 0.6698989898989899, + "eps": 0, + "loss/policy_avg": -0.23082350194454193, + "loss/value_avg": 0.9338909387588501, + "lr": 2.7110262568645063e-06, + "objective/entropy": -83.5711441040039, + "objective/kl": 42.80398941040039, + "objective/non_score_reward": -2.1401994228363037, + "objective/rlhf_reward": 13.555112838745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008490884792990983, + "policy/clipfrac_avg": 0.024211544543504715, + "policy/entropy_avg": 0.16208171844482422, + "step": 829, + "val/clipfrac_avg": 0.01703277789056301, + "val/num_eos_tokens": 407, + "val/ratio": 0.9889751672744751, + "val/ratio_var": 6.549843965331092e-05 + }, + { + "episode": 13280, + "epoch": 0.6707070707070707, + "eps": 0, + "loss/policy_avg": -0.1790890097618103, + "loss/value_avg": 0.9402360320091248, + "lr": 2.699159192863018e-06, + "objective/entropy": -91.52725219726562, + "objective/kl": 30.28129005432129, + "objective/non_score_reward": -1.5140645503997803, + "objective/rlhf_reward": 13.71249771118164, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0006465939804911613, + "policy/clipfrac_avg": 0.003674568608403206, + "policy/entropy_avg": 0.14063799381256104, + "step": 830, + "val/clipfrac_avg": 0.05747377499938011, + "val/num_eos_tokens": 250, + "val/ratio": 0.9919025897979736, + "val/ratio_var": 6.961268809391186e-05 + }, + { + "episode": 13296, + "epoch": 0.6715151515151515, + "eps": 0, + "loss/policy_avg": 0.3400639295578003, + "loss/value_avg": 1.9688762426376343, + "lr": 2.687308548795825e-06, + "objective/entropy": -54.39606475830078, + "objective/kl": 44.12242889404297, + "objective/non_score_reward": -2.2061214447021484, + "objective/rlhf_reward": 14.004816055297852, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009418983245268464, + "policy/clipfrac_avg": 0.009523827582597733, + "policy/entropy_avg": 0.18965661525726318, + "step": 831, + "val/clipfrac_avg": 0.045346200466156006, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880996942520142, + "val/ratio_var": 6.565819057868794e-05 + }, + { + "episode": 13312, + "epoch": 0.6723232323232323, + "eps": 0, + "loss/policy_avg": 0.11609108746051788, + "loss/value_avg": 0.5635161399841309, + "lr": 2.675474409234965e-06, + "objective/entropy": -66.36386108398438, + "objective/kl": 38.93806838989258, + "objective/non_score_reward": -1.9469034671783447, + "objective/rlhf_reward": 14.076534271240234, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008638559957034886, + "policy/clipfrac_avg": 0.005322727840393782, + "policy/entropy_avg": 0.15152549743652344, + "step": 832, + "val/clipfrac_avg": 0.04381619766354561, + "val/num_eos_tokens": 20, + "val/ratio": 0.9889363050460815, + "val/ratio_var": 3.002676021424122e-05 + }, + { + "episode": 13328, + "epoch": 0.6731313131313131, + "eps": 0, + "loss/policy_avg": -0.03308400511741638, + "loss/value_avg": 0.3210276961326599, + "lr": 2.66365685863469e-06, + "objective/entropy": -111.39958953857422, + "objective/kl": 47.16798400878906, + "objective/non_score_reward": -2.3583993911743164, + "objective/rlhf_reward": 11.7197265625, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009453936363570392, + "policy/clipfrac_avg": 0.004603059496730566, + "policy/entropy_avg": 0.18264532089233398, + "step": 833, + "val/clipfrac_avg": 0.062541663646698, + "val/num_eos_tokens": 763, + "val/ratio": 0.9879401922225952, + "val/ratio_var": 6.487595965154469e-05 + }, + { + "episode": 13344, + "epoch": 0.673939393939394, + "eps": 0, + "loss/policy_avg": 0.051825061440467834, + "loss/value_avg": 0.8246160745620728, + "lr": 2.651855981330872e-06, + "objective/entropy": -69.66492462158203, + "objective/kl": 37.68748474121094, + "objective/non_score_reward": -1.8843741416931152, + "objective/rlhf_reward": 12.764063835144043, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010055399034172297, + "policy/clipfrac_avg": 0.004349449649453163, + "policy/entropy_avg": 0.18214893341064453, + "step": 834, + "val/clipfrac_avg": 0.06374436616897583, + "val/num_eos_tokens": 38, + "val/ratio": 0.9867125749588013, + "val/ratio_var": 9.083253826247528e-05 + }, + { + "episode": 13360, + "epoch": 0.6747474747474748, + "eps": 0, + "loss/policy_avg": -0.043558381497859955, + "loss/value_avg": 0.6093795895576477, + "lr": 2.6400718615403852e-06, + "objective/entropy": -108.79554748535156, + "objective/kl": 39.288482666015625, + "objective/non_score_reward": -1.9644241333007812, + "objective/rlhf_reward": 14.504325866699219, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010659170802682638, + "policy/clipfrac_avg": 0.005707671865820885, + "policy/entropy_avg": 0.1986398696899414, + "step": 835, + "val/clipfrac_avg": 0.013371335342526436, + "val/num_eos_tokens": 647, + "val/ratio": 0.9869523048400879, + "val/ratio_var": 8.708865061635152e-05 + }, + { + "episode": 13376, + "epoch": 0.6755555555555556, + "eps": 0, + "loss/policy_avg": 0.09040576219558716, + "loss/value_avg": 0.4517725706100464, + "lr": 2.62830458336052e-06, + "objective/entropy": -59.1908073425293, + "objective/kl": 42.06044387817383, + "objective/non_score_reward": -2.1030220985412598, + "objective/rlhf_reward": 13.842290878295898, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001016529044136405, + "policy/clipfrac_avg": 0.007354079745709896, + "policy/entropy_avg": 0.20728302001953125, + "step": 836, + "val/clipfrac_avg": 0.030187882483005524, + "val/num_eos_tokens": 477, + "val/ratio": 0.9865027666091919, + "val/ratio_var": 5.405073534348048e-05 + }, + { + "episode": 13392, + "epoch": 0.6763636363636364, + "eps": 0, + "loss/policy_avg": -0.21371892094612122, + "loss/value_avg": 0.26992350816726685, + "lr": 2.6165542307683744e-06, + "objective/entropy": -99.01640319824219, + "objective/kl": 43.636474609375, + "objective/non_score_reward": -2.18182373046875, + "objective/rlhf_reward": 13.08380126953125, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009176525054499507, + "policy/clipfrac_avg": 0.006078221835196018, + "policy/entropy_avg": 0.1714763641357422, + "step": 837, + "val/clipfrac_avg": 0.025651777163147926, + "val/num_eos_tokens": 453, + "val/ratio": 0.988213300704956, + "val/ratio_var": 6.253684114199132e-05 + }, + { + "episode": 13408, + "epoch": 0.6771717171717172, + "eps": 0, + "loss/policy_avg": 0.08225658535957336, + "loss/value_avg": 0.4106031656265259, + "lr": 2.604820887620254e-06, + "objective/entropy": -62.21269989013672, + "objective/kl": 40.305843353271484, + "objective/non_score_reward": -2.015292167663574, + "objective/rlhf_reward": 15.75814437866211, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0009111049002967775, + "policy/clipfrac_avg": 0.005017759278416634, + "policy/entropy_avg": 0.18220901489257812, + "step": 838, + "val/clipfrac_avg": 0.0028512999415397644, + "val/num_eos_tokens": 268, + "val/ratio": 0.9878619909286499, + "val/ratio_var": 3.237010969314724e-05 + }, + { + "episode": 13424, + "epoch": 0.677979797979798, + "eps": 0, + "loss/policy_avg": 0.01355072483420372, + "loss/value_avg": 0.4876829981803894, + "lr": 2.5931046376510875e-06, + "objective/entropy": -38.304931640625, + "objective/kl": 35.24858093261719, + "objective/non_score_reward": -1.7624289989471436, + "objective/rlhf_reward": 12.284445762634277, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0008708706009201705, + "policy/clipfrac_avg": 0.005640837829560041, + "policy/entropy_avg": 0.16802191734313965, + "step": 839, + "val/clipfrac_avg": 0.029899097979068756, + "val/num_eos_tokens": 258, + "val/ratio": 0.989201009273529, + "val/ratio_var": 5.1689257816178724e-05 + }, + { + "episode": 13440, + "epoch": 0.6787878787878788, + "eps": 0, + "loss/policy_avg": 0.1623387336730957, + "loss/value_avg": 0.3584904074668884, + "lr": 2.5814055644738013e-06, + "objective/entropy": -63.923397064208984, + "objective/kl": 36.323509216308594, + "objective/non_score_reward": -1.8161752223968506, + "objective/rlhf_reward": 12.97288703918457, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009589061955921352, + "policy/clipfrac_avg": 0.006033012643456459, + "policy/entropy_avg": 0.17229270935058594, + "step": 840, + "val/clipfrac_avg": 0.11569125950336456, + "val/num_eos_tokens": 159, + "val/ratio": 0.988390326499939, + "val/ratio_var": 7.233954966068268e-05 + }, + { + "episode": 13456, + "epoch": 0.6795959595959596, + "eps": 0, + "loss/policy_avg": -0.29571330547332764, + "loss/value_avg": 0.5451829433441162, + "lr": 2.5697237515787564e-06, + "objective/entropy": -159.0351104736328, + "objective/kl": 34.90948486328125, + "objective/non_score_reward": -1.7454744577407837, + "objective/rlhf_reward": 13.145150184631348, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0007017417810857296, + "policy/clipfrac_avg": 0.005396063905209303, + "policy/entropy_avg": 0.12675732374191284, + "step": 841, + "val/clipfrac_avg": 0.019056372344493866, + "val/num_eos_tokens": 508, + "val/ratio": 0.9914215207099915, + "val/ratio_var": 8.740482735447586e-05 + }, + { + "episode": 13472, + "epoch": 0.6804040404040405, + "eps": 0, + "loss/policy_avg": -0.33980700373649597, + "loss/value_avg": 0.5796597003936768, + "lr": 2.5580592823331237e-06, + "objective/entropy": -83.5593490600586, + "objective/kl": 36.281490325927734, + "objective/non_score_reward": -1.8140745162963867, + "objective/rlhf_reward": 14.18592643737793, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009474118705838919, + "policy/clipfrac_avg": 0.004876702558249235, + "policy/entropy_avg": 0.14034080505371094, + "step": 842, + "val/clipfrac_avg": 0.03151373565196991, + "val/num_eos_tokens": 208, + "val/ratio": 0.9901444911956787, + "val/ratio_var": 6.344315625028685e-05 + }, + { + "episode": 13488, + "epoch": 0.6812121212121212, + "eps": 0, + "loss/policy_avg": 0.11847849935293198, + "loss/value_avg": 0.3053319454193115, + "lr": 2.5464122399803126e-06, + "objective/entropy": -74.54005432128906, + "objective/kl": 46.23011779785156, + "objective/non_score_reward": -2.3115057945251465, + "objective/rlhf_reward": 12.993181228637695, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009884861065074801, + "policy/clipfrac_avg": 0.006667924579232931, + "policy/entropy_avg": 0.19658470153808594, + "step": 843, + "val/clipfrac_avg": 0.00010080645006382838, + "val/num_eos_tokens": 52, + "val/ratio": 0.9878820180892944, + "val/ratio_var": 4.166604412603192e-05 + }, + { + "episode": 13504, + "epoch": 0.682020202020202, + "eps": 0, + "loss/policy_avg": 0.07477365434169769, + "loss/value_avg": 0.717211902141571, + "lr": 2.5347827076393505e-06, + "objective/entropy": -55.136016845703125, + "objective/kl": 36.799835205078125, + "objective/non_score_reward": -1.8399916887283325, + "objective/rlhf_reward": 12.995945930480957, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010256943060085177, + "policy/clipfrac_avg": 0.009538116864860058, + "policy/entropy_avg": 0.19795989990234375, + "step": 844, + "val/clipfrac_avg": 0.0769520252943039, + "val/num_eos_tokens": 573, + "val/ratio": 0.9861817955970764, + "val/ratio_var": 7.031750283204019e-05 + }, + { + "episode": 13520, + "epoch": 0.6828282828282828, + "eps": 0, + "loss/policy_avg": -0.010380454361438751, + "loss/value_avg": 0.8978620767593384, + "lr": 2.5231707683043204e-06, + "objective/entropy": -164.994873046875, + "objective/kl": 41.09418487548828, + "objective/non_score_reward": -2.0547094345092773, + "objective/rlhf_reward": 12.367165565490723, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006862258305773139, + "policy/clipfrac_avg": 0.005633420776575804, + "policy/entropy_avg": 0.136183500289917, + "step": 845, + "val/clipfrac_avg": 0.10761284828186035, + "val/num_eos_tokens": 289, + "val/ratio": 0.9907996654510498, + "val/ratio_var": 4.169603198533878e-05 + }, + { + "episode": 13536, + "epoch": 0.6836363636363636, + "eps": 0, + "loss/policy_avg": -0.0011942088603973389, + "loss/value_avg": 0.7805696129798889, + "lr": 2.5115765048437445e-06, + "objective/entropy": -119.26876068115234, + "objective/kl": 37.73551940917969, + "objective/non_score_reward": -1.8867759704589844, + "objective/rlhf_reward": 13.769474029541016, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0007093651220202446, + "policy/clipfrac_avg": 0.028275154531002045, + "policy/entropy_avg": 0.1421506404876709, + "step": 846, + "val/clipfrac_avg": 0.08967090398073196, + "val/num_eos_tokens": 760, + "val/ratio": 0.9911968111991882, + "val/ratio_var": 3.498927253531292e-05 + }, + { + "episode": 13552, + "epoch": 0.6844444444444444, + "eps": 0, + "loss/policy_avg": -0.007019490003585815, + "loss/value_avg": 0.8198627233505249, + "lr": 2.5000000000000015e-06, + "objective/entropy": -89.728271484375, + "objective/kl": 34.76280975341797, + "objective/non_score_reward": -1.7381404638290405, + "objective/rlhf_reward": 13.925922393798828, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009024562314152718, + "policy/clipfrac_avg": 0.007501127198338509, + "policy/entropy_avg": 0.17042064666748047, + "step": 847, + "val/clipfrac_avg": 0.028372831642627716, + "val/num_eos_tokens": 17, + "val/ratio": 0.9899512529373169, + "val/ratio_var": 7.211479532998055e-05 + }, + { + "episode": 13568, + "epoch": 0.6852525252525252, + "eps": 0, + "loss/policy_avg": 0.015100933611392975, + "loss/value_avg": 0.25284868478775024, + "lr": 2.488441336388737e-06, + "objective/entropy": -184.20425415039062, + "objective/kl": 46.20475769042969, + "objective/non_score_reward": -2.3102383613586426, + "objective/rlhf_reward": 11.728824615478516, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007638686802238226, + "policy/clipfrac_avg": 0.005230149254202843, + "policy/entropy_avg": 0.15384864807128906, + "step": 848, + "val/clipfrac_avg": 0.01555095985531807, + "val/num_eos_tokens": 404, + "val/ratio": 0.9898365139961243, + "val/ratio_var": 5.412843165686354e-05 + }, + { + "episode": 13584, + "epoch": 0.686060606060606, + "eps": 0, + "loss/policy_avg": -0.13074953854084015, + "loss/value_avg": 0.5254482626914978, + "lr": 2.4769005964982718e-06, + "objective/entropy": -70.96700286865234, + "objective/kl": 37.31745147705078, + "objective/non_score_reward": -1.8658727407455444, + "objective/rlhf_reward": 13.665377616882324, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000885509536601603, + "policy/clipfrac_avg": 0.006637534126639366, + "policy/entropy_avg": 0.17204856872558594, + "step": 849, + "val/clipfrac_avg": 0.050514861941337585, + "val/num_eos_tokens": 24, + "val/ratio": 0.9882212281227112, + "val/ratio_var": 5.447021248983219e-05 + }, + { + "episode": 13600, + "epoch": 0.6868686868686869, + "eps": 0, + "loss/policy_avg": 0.039310187101364136, + "loss/value_avg": 0.18585552275180817, + "lr": 2.465377862689019e-06, + "objective/entropy": -62.67339324951172, + "objective/kl": 48.39326477050781, + "objective/non_score_reward": -2.419663190841675, + "objective/rlhf_reward": 13.564711570739746, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009431451326236129, + "policy/clipfrac_avg": 0.004864457994699478, + "policy/entropy_avg": 0.19204330444335938, + "step": 850, + "val/clipfrac_avg": 0.01945144683122635, + "val/num_eos_tokens": 36, + "val/ratio": 0.9872653484344482, + "val/ratio_var": 3.194732926203869e-05 + }, + { + "episode": 13616, + "epoch": 0.6876767676767677, + "eps": 0, + "loss/policy_avg": 0.07975710183382034, + "loss/value_avg": 0.478973388671875, + "lr": 2.4538732171928847e-06, + "objective/entropy": -76.03438568115234, + "objective/kl": 35.364593505859375, + "objective/non_score_reward": -1.7682294845581055, + "objective/rlhf_reward": 15.075519561767578, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008544789161533117, + "policy/clipfrac_avg": 0.005153194069862366, + "policy/entropy_avg": 0.16641998291015625, + "step": 851, + "val/clipfrac_avg": 0.03764945641160011, + "val/num_eos_tokens": 298, + "val/ratio": 0.9884979724884033, + "val/ratio_var": 4.7100515075726435e-05 + }, + { + "episode": 13632, + "epoch": 0.6884848484848485, + "eps": 0, + "loss/policy_avg": -0.10211583226919174, + "loss/value_avg": 0.7199238538742065, + "lr": 2.4423867421126923e-06, + "objective/entropy": -59.744590759277344, + "objective/kl": 35.15203857421875, + "objective/non_score_reward": -1.7576020956039429, + "objective/rlhf_reward": 15.05489730834961, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009704260155558586, + "policy/clipfrac_avg": 0.004688786808401346, + "policy/entropy_avg": 0.17905235290527344, + "step": 852, + "val/clipfrac_avg": 0.17230291664600372, + "val/num_eos_tokens": 644, + "val/ratio": 0.9872374534606934, + "val/ratio_var": 4.29930878453888e-05 + }, + { + "episode": 13648, + "epoch": 0.6892929292929293, + "eps": 0, + "loss/policy_avg": 0.0028443392366170883, + "loss/value_avg": 0.3421381115913391, + "lr": 2.4309185194215885e-06, + "objective/entropy": -63.859352111816406, + "objective/kl": 41.87160110473633, + "objective/non_score_reward": -2.0935800075531006, + "objective/rlhf_reward": 16.01579475402832, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0011341955978423357, + "policy/clipfrac_avg": 0.005558031145483255, + "policy/entropy_avg": 0.21439552307128906, + "step": 853, + "val/clipfrac_avg": 0.07718860357999802, + "val/num_eos_tokens": 400, + "val/ratio": 0.9858847856521606, + "val/ratio_var": 5.448433512356132e-05 + }, + { + "episode": 13664, + "epoch": 0.6901010101010101, + "eps": 0, + "loss/policy_avg": 0.11498575657606125, + "loss/value_avg": 0.909265398979187, + "lr": 2.4194686309624664e-06, + "objective/entropy": -138.73922729492188, + "objective/kl": 39.14732360839844, + "objective/non_score_reward": -1.9573662281036377, + "objective/rlhf_reward": 11.542634010314941, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0011413603788241744, + "policy/clipfrac_avg": 0.008165586739778519, + "policy/entropy_avg": 0.18193864822387695, + "step": 854, + "val/clipfrac_avg": 0.04511754959821701, + "val/num_eos_tokens": 264, + "val/ratio": 0.9880124926567078, + "val/ratio_var": 7.85917291068472e-05 + }, + { + "episode": 13680, + "epoch": 0.6909090909090909, + "eps": 0, + "loss/policy_avg": 0.4261026978492737, + "loss/value_avg": 1.0351085662841797, + "lr": 2.408037158447375e-06, + "objective/entropy": -80.8604736328125, + "objective/kl": 39.87467956542969, + "objective/non_score_reward": -1.9937340021133423, + "objective/rlhf_reward": 12.850015640258789, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000854466634336859, + "policy/clipfrac_avg": 0.0031361770816147327, + "policy/entropy_avg": 0.1807117462158203, + "step": 855, + "val/clipfrac_avg": 0.06461638957262039, + "val/num_eos_tokens": 76, + "val/ratio": 0.9892903566360474, + "val/ratio_var": 6.158453470561653e-05 + }, + { + "episode": 13696, + "epoch": 0.6917171717171717, + "eps": 0, + "loss/policy_avg": 0.35515934228897095, + "loss/value_avg": 0.6004530787467957, + "lr": 2.3966241834569364e-06, + "objective/entropy": -109.00687408447266, + "objective/kl": 44.59175109863281, + "objective/non_score_reward": -2.2295875549316406, + "objective/rlhf_reward": 14.02041244506836, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008301680209115148, + "policy/clipfrac_avg": 0.004295141436159611, + "policy/entropy_avg": 0.16374778747558594, + "step": 856, + "val/clipfrac_avg": 0.0578775517642498, + "val/num_eos_tokens": 355, + "val/ratio": 0.9892109632492065, + "val/ratio_var": 3.649172867881134e-05 + }, + { + "episode": 13712, + "epoch": 0.6925252525252525, + "eps": 0, + "loss/policy_avg": -0.07683120667934418, + "loss/value_avg": 0.7507062554359436, + "lr": 2.385229787439768e-06, + "objective/entropy": -65.75347137451172, + "objective/kl": 33.233463287353516, + "objective/non_score_reward": -1.6616731882095337, + "objective/rlhf_reward": 12.885202407836914, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008445363491773605, + "policy/clipfrac_avg": 0.0028887931257486343, + "policy/entropy_avg": 0.1688997745513916, + "step": 857, + "val/clipfrac_avg": 0.029584139585494995, + "val/num_eos_tokens": 79, + "val/ratio": 0.9893004894256592, + "val/ratio_var": 4.4566000724444166e-05 + }, + { + "episode": 13728, + "epoch": 0.6933333333333334, + "eps": 0, + "loss/policy_avg": -0.04102202132344246, + "loss/value_avg": 0.295141339302063, + "lr": 2.3738540517118953e-06, + "objective/entropy": -137.21546936035156, + "objective/kl": 39.995941162109375, + "objective/non_score_reward": -1.9997971057891846, + "objective/rlhf_reward": 12.875202178955078, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009317261283285916, + "policy/clipfrac_avg": 0.016368409618735313, + "policy/entropy_avg": 0.18145465850830078, + "step": 858, + "val/clipfrac_avg": 0.02603679709136486, + "val/num_eos_tokens": 133, + "val/ratio": 0.9879333972930908, + "val/ratio_var": 7.448039104929194e-05 + }, + { + "episode": 13744, + "epoch": 0.6941414141414142, + "eps": 0, + "loss/policy_avg": -0.09325989335775375, + "loss/value_avg": 0.4778023660182953, + "lr": 2.3624970574561773e-06, + "objective/entropy": -126.39140319824219, + "objective/kl": 41.33527755737305, + "objective/non_score_reward": -2.0667638778686523, + "objective/rlhf_reward": 15.823860168457031, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009032993111759424, + "policy/clipfrac_avg": 0.008262147195637226, + "policy/entropy_avg": 0.1809234619140625, + "step": 859, + "val/clipfrac_avg": 0.006358179729431868, + "val/num_eos_tokens": 198, + "val/ratio": 0.9881711602210999, + "val/ratio_var": 8.430143498117104e-05 + }, + { + "episode": 13760, + "epoch": 0.694949494949495, + "eps": 0, + "loss/policy_avg": -0.1992878019809723, + "loss/value_avg": 0.4450816214084625, + "lr": 2.3511588857217228e-06, + "objective/entropy": -82.32437133789062, + "objective/kl": 36.61549377441406, + "objective/non_score_reward": -1.8307745456695557, + "objective/rlhf_reward": 14.497350692749023, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009405683958902955, + "policy/clipfrac_avg": 0.0031182412058115005, + "policy/entropy_avg": 0.1852397918701172, + "step": 860, + "val/clipfrac_avg": 0.04030741751194, + "val/num_eos_tokens": 674, + "val/ratio": 0.9874944686889648, + "val/ratio_var": 5.9595575294224545e-05 + }, + { + "episode": 13776, + "epoch": 0.6957575757575758, + "eps": 0, + "loss/policy_avg": -0.0025226399302482605, + "loss/value_avg": 0.6712653636932373, + "lr": 2.339839617423318e-06, + "objective/entropy": -97.76516723632812, + "objective/kl": 32.36253356933594, + "objective/non_score_reward": -1.6181267499923706, + "objective/rlhf_reward": 14.108436584472656, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008775125024840236, + "policy/clipfrac_avg": 0.006763250567018986, + "policy/entropy_avg": 0.15996170043945312, + "step": 861, + "val/clipfrac_avg": 0.01572282984852791, + "val/num_eos_tokens": 755, + "val/ratio": 0.9897935390472412, + "val/ratio_var": 4.971764428773895e-05 + }, + { + "episode": 13792, + "epoch": 0.6965656565656566, + "eps": 0, + "loss/policy_avg": 0.08292508125305176, + "loss/value_avg": 0.5537270307540894, + "lr": 2.32853933334084e-06, + "objective/entropy": -61.261680603027344, + "objective/kl": 36.61561584472656, + "objective/non_score_reward": -1.8307808637619019, + "objective/rlhf_reward": 12.762969970703125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009761417168192565, + "policy/clipfrac_avg": 0.0026826506946235895, + "policy/entropy_avg": 0.22123146057128906, + "step": 862, + "val/clipfrac_avg": 0.044523607939481735, + "val/num_eos_tokens": 56, + "val/ratio": 0.9871484041213989, + "val/ratio_var": 7.335093687288463e-05 + }, + { + "episode": 13808, + "epoch": 0.6973737373737374, + "eps": 0, + "loss/policy_avg": 0.36910420656204224, + "loss/value_avg": 0.9228214025497437, + "lr": 2.317258114118686e-06, + "objective/entropy": -76.75859069824219, + "objective/kl": 36.92033004760742, + "objective/non_score_reward": -1.8460166454315186, + "objective/rlhf_reward": 14.872733116149902, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009935328271239996, + "policy/clipfrac_avg": 0.006773949135094881, + "policy/entropy_avg": 0.1806344985961914, + "step": 863, + "val/clipfrac_avg": 0.07717607915401459, + "val/num_eos_tokens": 514, + "val/ratio": 0.9876723289489746, + "val/ratio_var": 6.279607623582706e-05 + }, + { + "episode": 13824, + "epoch": 0.6981818181818182, + "eps": 0, + "loss/policy_avg": -0.21535149216651917, + "loss/value_avg": 0.42513978481292725, + "lr": 2.305996040265198e-06, + "objective/entropy": -77.51583862304688, + "objective/kl": 45.520545959472656, + "objective/non_score_reward": -2.276027202606201, + "objective/rlhf_reward": 14.59897232055664, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008458065567538142, + "policy/clipfrac_avg": 0.005439429543912411, + "policy/entropy_avg": 0.16998863220214844, + "step": 864, + "val/clipfrac_avg": 0.03808614984154701, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894691109657288, + "val/ratio_var": 6.017706618877128e-05 + }, + { + "episode": 13840, + "epoch": 0.6989898989898989, + "eps": 0, + "loss/policy_avg": -0.08341407775878906, + "loss/value_avg": 0.7828294038772583, + "lr": 2.29475319215209e-06, + "objective/entropy": -82.54826354980469, + "objective/kl": 39.21794891357422, + "objective/non_score_reward": -1.9608973264694214, + "objective/rlhf_reward": 14.359415054321289, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007373163243755698, + "policy/clipfrac_avg": 0.003970882389694452, + "policy/entropy_avg": 0.14882469177246094, + "step": 865, + "val/clipfrac_avg": 0.17675819993019104, + "val/num_eos_tokens": 134, + "val/ratio": 0.9905297756195068, + "val/ratio_var": 5.841355596203357e-05 + }, + { + "episode": 13856, + "epoch": 0.6997979797979798, + "eps": 0, + "loss/policy_avg": 0.029253050684928894, + "loss/value_avg": 0.8540998101234436, + "lr": 2.2835296500138664e-06, + "objective/entropy": -121.09165954589844, + "objective/kl": 34.1153564453125, + "objective/non_score_reward": -1.7057678699493408, + "objective/rlhf_reward": 13.333294868469238, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009264532127417624, + "policy/clipfrac_avg": 0.0050398847088217735, + "policy/entropy_avg": 0.18099594116210938, + "step": 866, + "val/clipfrac_avg": 0.02329857461154461, + "val/num_eos_tokens": 1353, + "val/ratio": 0.9883856177330017, + "val/ratio_var": 8.088548202067614e-05 + }, + { + "episode": 13872, + "epoch": 0.7006060606060606, + "eps": 0, + "loss/policy_avg": -0.1498204916715622, + "loss/value_avg": 0.4047302007675171, + "lr": 2.272325493947257e-06, + "objective/entropy": -49.65830993652344, + "objective/kl": 36.36531066894531, + "objective/non_score_reward": -1.8182655572891235, + "objective/rlhf_reward": 14.494234085083008, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010429773246869445, + "policy/clipfrac_avg": 0.005474097095429897, + "policy/entropy_avg": 0.18477344512939453, + "step": 867, + "val/clipfrac_avg": 0.0032526040449738503, + "val/num_eos_tokens": 101, + "val/ratio": 0.9877700805664062, + "val/ratio_var": 9.155944280792028e-05 + }, + { + "episode": 13888, + "epoch": 0.7014141414141414, + "eps": 0, + "loss/policy_avg": -0.1165279895067215, + "loss/value_avg": 0.7669879198074341, + "lr": 2.261140803910644e-06, + "objective/entropy": -77.34773254394531, + "objective/kl": 38.648643493652344, + "objective/non_score_reward": -1.9324320554733276, + "objective/rlhf_reward": 13.637880325317383, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000937658129259944, + "policy/clipfrac_avg": 0.006235727574676275, + "policy/entropy_avg": 0.19645977020263672, + "step": 868, + "val/clipfrac_avg": 0.058441728353500366, + "val/num_eos_tokens": 310, + "val/ratio": 0.9879204034805298, + "val/ratio_var": 6.414015660993755e-05 + }, + { + "episode": 13904, + "epoch": 0.7022222222222222, + "eps": 0, + "loss/policy_avg": -0.016327757388353348, + "loss/value_avg": 0.43844515085220337, + "lr": 2.2499756597234872e-06, + "objective/entropy": -163.28765869140625, + "objective/kl": 48.81157684326172, + "objective/non_score_reward": -2.4405789375305176, + "objective/rlhf_reward": 11.70395278930664, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.000844982685521245, + "policy/clipfrac_avg": 0.0032518147490918636, + "policy/entropy_avg": 0.16161727905273438, + "step": 869, + "val/clipfrac_avg": 0.014801801182329655, + "val/num_eos_tokens": 1949, + "val/ratio": 0.9885438680648804, + "val/ratio_var": 5.214377597440034e-05 + }, + { + "episode": 13920, + "epoch": 0.703030303030303, + "eps": 0, + "loss/policy_avg": -0.18964827060699463, + "loss/value_avg": 0.5136280655860901, + "lr": 2.238830141065765e-06, + "objective/entropy": -90.82677459716797, + "objective/kl": 40.3735237121582, + "objective/non_score_reward": -2.0186760425567627, + "objective/rlhf_reward": 15.153199195861816, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008435281342826784, + "policy/clipfrac_avg": 0.005789198447018862, + "policy/entropy_avg": 0.1680126190185547, + "step": 870, + "val/clipfrac_avg": 0.0648677870631218, + "val/num_eos_tokens": 555, + "val/ratio": 0.9890276193618774, + "val/ratio_var": 7.125848060240969e-05 + }, + { + "episode": 13936, + "epoch": 0.7038383838383838, + "eps": 0, + "loss/policy_avg": 0.02698252536356449, + "loss/value_avg": 0.2791609764099121, + "lr": 2.2277043274773856e-06, + "objective/entropy": -45.07421112060547, + "objective/kl": 39.21965026855469, + "objective/non_score_reward": -1.9609825611114502, + "objective/rlhf_reward": 14.624954223632812, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009133699350059032, + "policy/clipfrac_avg": 0.004030972719192505, + "policy/entropy_avg": 0.18420028686523438, + "step": 871, + "val/clipfrac_avg": 0.03648557886481285, + "val/num_eos_tokens": 232, + "val/ratio": 0.9881106615066528, + "val/ratio_var": 5.0894170271931216e-05 + }, + { + "episode": 13952, + "epoch": 0.7046464646464646, + "eps": 0, + "loss/policy_avg": -0.12599462270736694, + "loss/value_avg": 0.5007761716842651, + "lr": 2.216598298357646e-06, + "objective/entropy": -52.43424987792969, + "objective/kl": 37.084686279296875, + "objective/non_score_reward": -1.8542344570159912, + "objective/rlhf_reward": 14.18482780456543, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008421384263783693, + "policy/clipfrac_avg": 0.002664695493876934, + "policy/entropy_avg": 0.1716923713684082, + "step": 872, + "val/clipfrac_avg": 0.0010017228778451681, + "val/num_eos_tokens": 274, + "val/ratio": 0.9881088137626648, + "val/ratio_var": 6.610793934669346e-05 + }, + { + "episode": 13968, + "epoch": 0.7054545454545454, + "eps": 0, + "loss/policy_avg": 0.35178622603416443, + "loss/value_avg": 0.5846590995788574, + "lr": 2.2055121329646416e-06, + "objective/entropy": -138.92434692382812, + "objective/kl": 41.75783157348633, + "objective/non_score_reward": -2.0878915786743164, + "objective/rlhf_reward": 13.091796875, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008433774346485734, + "policy/clipfrac_avg": 0.006052501033991575, + "policy/entropy_avg": 0.16450119018554688, + "step": 873, + "val/clipfrac_avg": 0.0322103314101696, + "val/num_eos_tokens": 1144, + "val/ratio": 0.9888837337493896, + "val/ratio_var": 5.522933497559279e-05 + }, + { + "episode": 13984, + "epoch": 0.7062626262626263, + "eps": 0, + "loss/policy_avg": -0.004525661468505859, + "loss/value_avg": 0.5259770750999451, + "lr": 2.1944459104147188e-06, + "objective/entropy": -122.14696502685547, + "objective/kl": 37.81830978393555, + "objective/non_score_reward": -1.8909155130386353, + "objective/rlhf_reward": 13.296585083007812, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0006640740903094411, + "policy/clipfrac_avg": 0.004708480089902878, + "policy/entropy_avg": 0.14041614532470703, + "step": 874, + "val/clipfrac_avg": 0.10728074610233307, + "val/num_eos_tokens": 35, + "val/ratio": 0.990477979183197, + "val/ratio_var": 6.866270996397361e-05 + }, + { + "episode": 14000, + "epoch": 0.7070707070707071, + "eps": 0, + "loss/policy_avg": 0.11862924695014954, + "loss/value_avg": 1.0722770690917969, + "lr": 2.1833997096818897e-06, + "objective/entropy": -54.207332611083984, + "objective/kl": 37.62938690185547, + "objective/non_score_reward": -1.8814692497253418, + "objective/rlhf_reward": 14.2200927734375, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000859625346492976, + "policy/clipfrac_avg": 0.005390159785747528, + "policy/entropy_avg": 0.1755199432373047, + "step": 875, + "val/clipfrac_avg": 0.0593750923871994, + "val/num_eos_tokens": 201, + "val/ratio": 0.9878208637237549, + "val/ratio_var": 6.484645564341918e-05 + }, + { + "episode": 14016, + "epoch": 0.7078787878787879, + "eps": 0, + "loss/policy_avg": -0.2030085027217865, + "loss/value_avg": 0.5594953298568726, + "lr": 2.1723736095972946e-06, + "objective/entropy": -16.795242309570312, + "objective/kl": 44.697776794433594, + "objective/non_score_reward": -2.234889030456543, + "objective/rlhf_reward": 13.140110969543457, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0011914006900042295, + "policy/clipfrac_avg": 0.005575358867645264, + "policy/entropy_avg": 0.2543792724609375, + "step": 876, + "val/clipfrac_avg": 0.02393021248281002, + "val/num_eos_tokens": 24, + "val/ratio": 0.9835648536682129, + "val/ratio_var": 0.00010361165914218873 + }, + { + "episode": 14032, + "epoch": 0.7086868686868687, + "eps": 0, + "loss/policy_avg": 0.05069049820303917, + "loss/value_avg": 0.6454532146453857, + "lr": 2.161367688848618e-06, + "objective/entropy": -105.85357666015625, + "objective/kl": 50.78942108154297, + "objective/non_score_reward": -2.53947114944458, + "objective/rlhf_reward": 13.077716827392578, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008386933477595448, + "policy/clipfrac_avg": 0.003330180887132883, + "policy/entropy_avg": 0.1841726303100586, + "step": 877, + "val/clipfrac_avg": 0.010819870047271252, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886084198951721, + "val/ratio_var": 5.4145024478202686e-05 + }, + { + "episode": 14048, + "epoch": 0.7094949494949495, + "eps": 0, + "loss/policy_avg": -0.32361558079719543, + "loss/value_avg": 0.4275820851325989, + "lr": 2.1503820259795357e-06, + "objective/entropy": -193.71432495117188, + "objective/kl": 41.63684844970703, + "objective/non_score_reward": -2.0818424224853516, + "objective/rlhf_reward": 14.261907577514648, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007008577231317759, + "policy/clipfrac_avg": 0.004103646147996187, + "policy/entropy_avg": 0.1445927619934082, + "step": 878, + "val/clipfrac_avg": 7.62939453125e-06, + "val/num_eos_tokens": 323, + "val/ratio": 0.9903497695922852, + "val/ratio_var": 3.22636442433577e-05 + }, + { + "episode": 14064, + "epoch": 0.7103030303030303, + "eps": 0, + "loss/policy_avg": -0.1645442247390747, + "loss/value_avg": 0.7777354717254639, + "lr": 2.139416699389153e-06, + "objective/entropy": -45.89549255371094, + "objective/kl": 34.282958984375, + "objective/non_score_reward": -1.7141478061676025, + "objective/rlhf_reward": 14.692102432250977, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007650890620425344, + "policy/clipfrac_avg": 0.00561137031763792, + "policy/entropy_avg": 0.15566253662109375, + "step": 879, + "val/clipfrac_avg": 0.10208053886890411, + "val/num_eos_tokens": 49, + "val/ratio": 0.9895084500312805, + "val/ratio_var": 3.380152702447958e-05 + }, + { + "episode": 14080, + "epoch": 0.7111111111111111, + "eps": 0, + "loss/policy_avg": -0.023053357377648354, + "loss/value_avg": 0.521371603012085, + "lr": 2.128471787331446e-06, + "objective/entropy": -50.96327209472656, + "objective/kl": 41.30862808227539, + "objective/non_score_reward": -2.0654313564300537, + "objective/rlhf_reward": 14.450193405151367, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009377336828038096, + "policy/clipfrac_avg": 0.00421791011467576, + "policy/entropy_avg": 0.17507076263427734, + "step": 880, + "val/clipfrac_avg": 0.04448426887392998, + "val/num_eos_tokens": 368, + "val/ratio": 0.9889349937438965, + "val/ratio_var": 3.734078927664086e-05 + }, + { + "episode": 14096, + "epoch": 0.711919191919192, + "eps": 0, + "loss/policy_avg": -0.07007278501987457, + "loss/value_avg": 0.2162141352891922, + "lr": 2.117547367914705e-06, + "objective/entropy": -69.7264175415039, + "objective/kl": 41.05116653442383, + "objective/non_score_reward": -2.052558422088623, + "objective/rlhf_reward": 14.830253601074219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.001040782080963254, + "policy/clipfrac_avg": 0.0033751465380191803, + "policy/entropy_avg": 0.20317459106445312, + "step": 881, + "val/clipfrac_avg": 0.03193364292383194, + "val/num_eos_tokens": 551, + "val/ratio": 0.9861003160476685, + "val/ratio_var": 8.361194340977818e-05 + }, + { + "episode": 14112, + "epoch": 0.7127272727272728, + "eps": 0, + "loss/policy_avg": -0.07847104221582413, + "loss/value_avg": 0.2657424807548523, + "lr": 2.1066435191009717e-06, + "objective/entropy": -86.86426544189453, + "objective/kl": 45.00962829589844, + "objective/non_score_reward": -2.250481367111206, + "objective/rlhf_reward": 13.913580894470215, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010606081923469901, + "policy/clipfrac_avg": 0.0069802612997591496, + "policy/entropy_avg": 0.20689773559570312, + "step": 882, + "val/clipfrac_avg": 0.038475796580314636, + "val/num_eos_tokens": 546, + "val/ratio": 0.985522449016571, + "val/ratio_var": 8.580441499361768e-05 + }, + { + "episode": 14128, + "epoch": 0.7135353535353536, + "eps": 0, + "loss/policy_avg": -0.1323227733373642, + "loss/value_avg": 0.21754738688468933, + "lr": 2.095760318705487e-06, + "objective/entropy": -109.19290161132812, + "objective/kl": 42.61073303222656, + "objective/non_score_reward": -2.1305365562438965, + "objective/rlhf_reward": 14.447587966918945, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010359378065913916, + "policy/clipfrac_avg": 0.006892787292599678, + "policy/entropy_avg": 0.20017433166503906, + "step": 883, + "val/clipfrac_avg": 0.010656789876520634, + "val/num_eos_tokens": 36, + "val/ratio": 0.9864689111709595, + "val/ratio_var": 4.844724026042968e-05 + }, + { + "episode": 14144, + "epoch": 0.7143434343434344, + "eps": 0, + "loss/policy_avg": -0.1507091522216797, + "loss/value_avg": 0.2665415108203888, + "lr": 2.0848978443961314e-06, + "objective/entropy": -119.52471923828125, + "objective/kl": 43.517696380615234, + "objective/non_score_reward": -2.17588472366333, + "objective/rlhf_reward": 14.300678253173828, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000938809709623456, + "policy/clipfrac_avg": 0.012131594121456146, + "policy/entropy_avg": 0.18040943145751953, + "step": 884, + "val/clipfrac_avg": 0.006075765937566757, + "val/num_eos_tokens": 620, + "val/ratio": 0.9873965978622437, + "val/ratio_var": 6.621635111514479e-05 + }, + { + "episode": 14160, + "epoch": 0.7151515151515152, + "eps": 0, + "loss/policy_avg": -0.028530046343803406, + "loss/value_avg": 0.434810072183609, + "lr": 2.074056173692881e-06, + "objective/entropy": -62.273380279541016, + "objective/kl": 45.77116394042969, + "objective/non_score_reward": -2.2885584831237793, + "objective/rlhf_reward": 14.617691040039062, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009718666551634669, + "policy/clipfrac_avg": 0.004924894310534, + "policy/entropy_avg": 0.20779800415039062, + "step": 885, + "val/clipfrac_avg": 0.036309417337179184, + "val/num_eos_tokens": 179, + "val/ratio": 0.9870821237564087, + "val/ratio_var": 8.347785478690639e-05 + }, + { + "episode": 14176, + "epoch": 0.7159595959595959, + "eps": 0, + "loss/policy_avg": -0.15033547580242157, + "loss/value_avg": 0.6043248772621155, + "lr": 2.0632353839672415e-06, + "objective/entropy": -84.31056213378906, + "objective/kl": 32.41828918457031, + "objective/non_score_reward": -1.6209145784378052, + "objective/rlhf_reward": 15.332210540771484, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008589382632635534, + "policy/clipfrac_avg": 0.002752305706962943, + "policy/entropy_avg": 0.18497180938720703, + "step": 886, + "val/clipfrac_avg": 0.03947104513645172, + "val/num_eos_tokens": 180, + "val/ratio": 0.9879313707351685, + "val/ratio_var": 5.601351585937664e-05 + }, + { + "episode": 14192, + "epoch": 0.7167676767676767, + "eps": 0, + "loss/policy_avg": 0.2279771864414215, + "loss/value_avg": 1.2074341773986816, + "lr": 2.0524355524417017e-06, + "objective/entropy": -51.082984924316406, + "objective/kl": 36.84368133544922, + "objective/non_score_reward": -1.842184066772461, + "objective/rlhf_reward": 14.087503433227539, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010461971396580338, + "policy/clipfrac_avg": 0.004587469156831503, + "policy/entropy_avg": 0.22597980499267578, + "step": 887, + "val/clipfrac_avg": 0.015519710257649422, + "val/num_eos_tokens": 448, + "val/ratio": 0.9849185943603516, + "val/ratio_var": 6.34926327620633e-05 + }, + { + "episode": 14208, + "epoch": 0.7175757575757575, + "eps": 0, + "loss/policy_avg": -0.026267291978001595, + "loss/value_avg": 0.5965927839279175, + "lr": 2.041656756189184e-06, + "objective/entropy": -154.47012329101562, + "objective/kl": 42.720699310302734, + "objective/non_score_reward": -2.1360349655151367, + "objective/rlhf_reward": 14.957715034484863, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009299394441768527, + "policy/clipfrac_avg": 0.008321288973093033, + "policy/entropy_avg": 0.1751893162727356, + "step": 888, + "val/clipfrac_avg": 0.01875000074505806, + "val/num_eos_tokens": 428, + "val/ratio": 0.9883604049682617, + "val/ratio_var": 7.325677870539948e-05 + }, + { + "episode": 14224, + "epoch": 0.7183838383838383, + "eps": 0, + "loss/policy_avg": 0.40078702569007874, + "loss/value_avg": 1.1534854173660278, + "lr": 2.030899072132493e-06, + "objective/entropy": -128.48831176757812, + "objective/kl": 33.898155212402344, + "objective/non_score_reward": -1.6949079036712646, + "objective/rlhf_reward": 12.598060607910156, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006705562118440866, + "policy/clipfrac_avg": 0.005831941030919552, + "policy/entropy_avg": 0.21662330627441406, + "step": 889, + "val/clipfrac_avg": 0.02483724057674408, + "val/num_eos_tokens": 47, + "val/ratio": 0.9912885427474976, + "val/ratio_var": 7.058407936710864e-05 + }, + { + "episode": 14240, + "epoch": 0.7191919191919192, + "eps": 0, + "loss/policy_avg": -0.0855930745601654, + "loss/value_avg": 0.32921552658081055, + "lr": 2.0201625770437647e-06, + "objective/entropy": -86.9806137084961, + "objective/kl": 40.06652069091797, + "objective/non_score_reward": -2.003325939178467, + "objective/rlhf_reward": 14.035737037658691, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008815613109618425, + "policy/clipfrac_avg": 0.006017718929797411, + "policy/entropy_avg": 0.1755352020263672, + "step": 890, + "val/clipfrac_avg": 0.06256753951311111, + "val/num_eos_tokens": 20, + "val/ratio": 0.9872984290122986, + "val/ratio_var": 8.527067984687164e-05 + }, + { + "episode": 14256, + "epoch": 0.72, + "eps": 0, + "loss/policy_avg": -0.17781905829906464, + "loss/value_avg": 0.2275725156068802, + "lr": 2.00944734754392e-06, + "objective/entropy": -99.16207122802734, + "objective/kl": 41.95650863647461, + "objective/non_score_reward": -2.097825288772583, + "objective/rlhf_reward": 13.74592399597168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007897773757576942, + "policy/clipfrac_avg": 0.0043031517416238785, + "policy/entropy_avg": 0.1585559844970703, + "step": 891, + "val/clipfrac_avg": 0.012334126979112625, + "val/num_eos_tokens": 424, + "val/ratio": 0.9894897937774658, + "val/ratio_var": 5.430886085378006e-05 + }, + { + "episode": 14272, + "epoch": 0.7208080808080808, + "eps": 0, + "loss/policy_avg": -0.36396488547325134, + "loss/value_avg": 0.8662234544754028, + "lr": 1.9987534601021225e-06, + "objective/entropy": -61.42802047729492, + "objective/kl": 37.07469177246094, + "objective/non_score_reward": -1.8537347316741943, + "objective/rlhf_reward": 13.044702529907227, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008486385922878981, + "policy/clipfrac_avg": 0.006757801398634911, + "policy/entropy_avg": 0.1866321563720703, + "step": 892, + "val/clipfrac_avg": 0.054801926016807556, + "val/num_eos_tokens": 584, + "val/ratio": 0.9883260726928711, + "val/ratio_var": 6.7303488322068e-05 + }, + { + "episode": 14288, + "epoch": 0.7216161616161616, + "eps": 0, + "loss/policy_avg": -0.12171509861946106, + "loss/value_avg": 0.3499939441680908, + "lr": 1.988080991035223e-06, + "objective/entropy": -58.38874053955078, + "objective/kl": 42.03372573852539, + "objective/non_score_reward": -2.1016862392425537, + "objective/rlhf_reward": 12.828001022338867, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008078413084149361, + "policy/clipfrac_avg": 0.004710954613983631, + "policy/entropy_avg": 0.16219139099121094, + "step": 893, + "val/clipfrac_avg": 0.03385913744568825, + "val/num_eos_tokens": 211, + "val/ratio": 0.9909918904304504, + "val/ratio_var": 4.974031253368594e-05 + }, + { + "episode": 14304, + "epoch": 0.7224242424242424, + "eps": 0, + "loss/policy_avg": 0.15615974366664886, + "loss/value_avg": 0.4985978901386261, + "lr": 1.977430016507222e-06, + "objective/entropy": -72.03929138183594, + "objective/kl": 43.86260986328125, + "objective/non_score_reward": -2.1931304931640625, + "objective/rlhf_reward": 13.322494506835938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009859772399067879, + "policy/clipfrac_avg": 0.004861406981945038, + "policy/entropy_avg": 0.19806480407714844, + "step": 894, + "val/clipfrac_avg": 0.008869262412190437, + "val/num_eos_tokens": 389, + "val/ratio": 0.9869673252105713, + "val/ratio_var": 5.9525649703573436e-05 + }, + { + "episode": 14320, + "epoch": 0.7232323232323232, + "eps": 0, + "loss/policy_avg": -0.018653089180588722, + "loss/value_avg": 0.3236011266708374, + "lr": 1.966800612528723e-06, + "objective/entropy": -30.8804874420166, + "objective/kl": 38.795677185058594, + "objective/non_score_reward": -1.9397838115692139, + "objective/rlhf_reward": 12.958653450012207, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011067797895520926, + "policy/clipfrac_avg": 0.004753993824124336, + "policy/entropy_avg": 0.21401214599609375, + "step": 895, + "val/clipfrac_avg": 0.013285137712955475, + "val/num_eos_tokens": 190, + "val/ratio": 0.9844212532043457, + "val/ratio_var": 5.578770287684165e-05 + }, + { + "episode": 14336, + "epoch": 0.724040404040404, + "eps": 0, + "loss/policy_avg": -0.04005177319049835, + "loss/value_avg": 0.564488410949707, + "lr": 1.956192854956397e-06, + "objective/entropy": -88.01570129394531, + "objective/kl": 40.9593505859375, + "objective/non_score_reward": -2.0479674339294434, + "objective/rlhf_reward": 13.350469589233398, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010309319477528334, + "policy/clipfrac_avg": 0.005483411252498627, + "policy/entropy_avg": 0.21037864685058594, + "step": 896, + "val/clipfrac_avg": 0.01046415138989687, + "val/num_eos_tokens": 539, + "val/ratio": 0.9865807890892029, + "val/ratio_var": 9.776368096936494e-05 + }, + { + "episode": 14352, + "epoch": 0.7248484848484849, + "eps": 0, + "loss/policy_avg": -0.1358407437801361, + "loss/value_avg": 0.536889910697937, + "lr": 1.945606819492429e-06, + "objective/entropy": -98.2094955444336, + "objective/kl": 37.47423553466797, + "objective/non_score_reward": -1.8737118244171143, + "objective/rlhf_reward": 13.634100914001465, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007985340780578554, + "policy/clipfrac_avg": 0.005063176155090332, + "policy/entropy_avg": 0.14582359790802002, + "step": 897, + "val/clipfrac_avg": 0.06062069162726402, + "val/num_eos_tokens": 1182, + "val/ratio": 0.9903557300567627, + "val/ratio_var": 5.06751166540198e-05 + }, + { + "episode": 14368, + "epoch": 0.7256565656565657, + "eps": 0, + "loss/policy_avg": 0.011427253484725952, + "loss/value_avg": 0.263162761926651, + "lr": 1.9350425816839875e-06, + "objective/entropy": -110.30941772460938, + "objective/kl": 41.13695526123047, + "objective/non_score_reward": -2.0568478107452393, + "objective/rlhf_reward": 13.72440242767334, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011626393534243107, + "policy/clipfrac_avg": 0.006754214409738779, + "policy/entropy_avg": 0.2226581573486328, + "step": 898, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 10, + "val/ratio": 0.9850178956985474, + "val/ratio_var": 8.876781794242561e-05 + }, + { + "episode": 14384, + "epoch": 0.7264646464646465, + "eps": 0, + "loss/policy_avg": 0.11541193723678589, + "loss/value_avg": 0.4631091356277466, + "lr": 1.9245002169226814e-06, + "objective/entropy": -101.70489501953125, + "objective/kl": 48.557960510253906, + "objective/non_score_reward": -2.4278979301452637, + "objective/rlhf_reward": 13.314289093017578, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009826980531215668, + "policy/clipfrac_avg": 0.004610124975442886, + "policy/entropy_avg": 0.1724681854248047, + "step": 899, + "val/clipfrac_avg": 0.013620924204587936, + "val/num_eos_tokens": 516, + "val/ratio": 0.9876973628997803, + "val/ratio_var": 5.4934793297434226e-05 + }, + { + "episode": 14400, + "epoch": 0.7272727272727273, + "eps": 0, + "loss/policy_avg": 0.07936800271272659, + "loss/value_avg": 0.6616516709327698, + "lr": 1.913979800444021e-06, + "objective/entropy": -91.96019744873047, + "objective/kl": 41.26939010620117, + "objective/non_score_reward": -2.063469409942627, + "objective/rlhf_reward": 12.522467613220215, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0009332017507404089, + "policy/clipfrac_avg": 0.0043076397851109505, + "policy/entropy_avg": 0.18123245239257812, + "step": 900, + "val/clipfrac_avg": 0.01476163137704134, + "val/num_eos_tokens": 595, + "val/ratio": 0.9875973463058472, + "val/ratio_var": 7.475823076674715e-05 + }, + { + "episode": 14416, + "epoch": 0.7280808080808081, + "eps": 0, + "loss/policy_avg": -0.02671997621655464, + "loss/value_avg": 0.7174835205078125, + "lr": 1.9034814073268898e-06, + "objective/entropy": -59.236602783203125, + "objective/kl": 39.73054122924805, + "objective/non_score_reward": -1.9865270853042603, + "objective/rlhf_reward": 12.435348510742188, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011704354546964169, + "policy/clipfrac_avg": 0.004937891382724047, + "policy/entropy_avg": 0.22887516021728516, + "step": 901, + "val/clipfrac_avg": 0.13261640071868896, + "val/num_eos_tokens": 121, + "val/ratio": 0.9848372936248779, + "val/ratio_var": 8.883672853698954e-05 + }, + { + "episode": 14432, + "epoch": 0.7288888888888889, + "eps": 0, + "loss/policy_avg": 0.05788304656744003, + "loss/value_avg": 0.2591496407985687, + "lr": 1.8930051124929904e-06, + "objective/entropy": -53.46650695800781, + "objective/kl": 46.268035888671875, + "objective/non_score_reward": -2.313401937484741, + "objective/rlhf_reward": 14.38972282409668, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010022292844951153, + "policy/clipfrac_avg": 0.004066841676831245, + "policy/entropy_avg": 0.20316123962402344, + "step": 902, + "val/clipfrac_avg": 0.013965151272714138, + "val/num_eos_tokens": 455, + "val/ratio": 0.9866280555725098, + "val/ratio_var": 5.229823727859184e-05 + }, + { + "episode": 14448, + "epoch": 0.7296969696969697, + "eps": 0, + "loss/policy_avg": -0.11720985174179077, + "loss/value_avg": 0.23323997855186462, + "lr": 1.8825509907063328e-06, + "objective/entropy": -65.6883773803711, + "objective/kl": 46.209991455078125, + "objective/non_score_reward": -2.310499668121338, + "objective/rlhf_reward": 16.78325080871582, + "objective/scores": 19.125, + "policy/approxkl_avg": 0.0010066109243780375, + "policy/clipfrac_avg": 0.005406823009252548, + "policy/entropy_avg": 0.20101547241210938, + "step": 903, + "val/clipfrac_avg": 0.015051587484776974, + "val/num_eos_tokens": 205, + "val/ratio": 0.9865299463272095, + "val/ratio_var": 5.01565846207086e-05 + }, + { + "episode": 14464, + "epoch": 0.7305050505050505, + "eps": 0, + "loss/policy_avg": -0.4681462049484253, + "loss/value_avg": 0.8052223920822144, + "lr": 1.87211911657268e-06, + "objective/entropy": -118.0733642578125, + "objective/kl": 37.29925537109375, + "objective/non_score_reward": -1.8649628162384033, + "objective/rlhf_reward": 15.885037422180176, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0007588897133246064, + "policy/clipfrac_avg": 0.0023906652349978685, + "policy/entropy_avg": 0.15973949432373047, + "step": 904, + "val/clipfrac_avg": 0.13810205459594727, + "val/num_eos_tokens": 155, + "val/ratio": 0.9902135729789734, + "val/ratio_var": 7.25651771062985e-05 + }, + { + "episode": 14480, + "epoch": 0.7313131313131314, + "eps": 0, + "loss/policy_avg": 0.19160020351409912, + "loss/value_avg": 0.9071890115737915, + "lr": 1.8617095645390332e-06, + "objective/entropy": -22.496440887451172, + "objective/kl": 39.160484313964844, + "objective/non_score_reward": -1.958024263381958, + "objective/rlhf_reward": 12.323225975036621, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0010718535631895065, + "policy/clipfrac_avg": 0.004276500083506107, + "policy/entropy_avg": 0.19223785400390625, + "step": 905, + "val/clipfrac_avg": 0.03053325042128563, + "val/num_eos_tokens": 240, + "val/ratio": 0.9871090650558472, + "val/ratio_var": 4.5721961214439943e-05 + }, + { + "episode": 14496, + "epoch": 0.7321212121212122, + "eps": 0, + "loss/policy_avg": -0.11877916008234024, + "loss/value_avg": 0.6873422265052795, + "lr": 1.8513224088930814e-06, + "objective/entropy": -52.41043472290039, + "objective/kl": 30.631162643432617, + "objective/non_score_reward": -1.5315582752227783, + "objective/rlhf_reward": 16.296566009521484, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.001107763615436852, + "policy/clipfrac_avg": 0.006524171680212021, + "policy/entropy_avg": 0.2064342498779297, + "step": 906, + "val/clipfrac_avg": 0.000294811325147748, + "val/num_eos_tokens": 261, + "val/ratio": 0.9861389398574829, + "val/ratio_var": 6.195842433953658e-05 + }, + { + "episode": 14512, + "epoch": 0.7329292929292929, + "eps": 0, + "loss/policy_avg": 0.10565789043903351, + "loss/value_avg": 0.5067005157470703, + "lr": 1.8409577237626935e-06, + "objective/entropy": -61.856544494628906, + "objective/kl": 43.33234405517578, + "objective/non_score_reward": -2.1666173934936523, + "objective/rlhf_reward": 13.145882606506348, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010527942795306444, + "policy/clipfrac_avg": 0.004575061611831188, + "policy/entropy_avg": 0.18899154663085938, + "step": 907, + "val/clipfrac_avg": 0.005580357275903225, + "val/num_eos_tokens": 496, + "val/ratio": 0.9872788190841675, + "val/ratio_var": 7.066375110298395e-05 + }, + { + "episode": 14528, + "epoch": 0.7337373737373737, + "eps": 0, + "loss/policy_avg": 0.04191252216696739, + "loss/value_avg": 0.30707108974456787, + "lr": 1.8306155831153699e-06, + "objective/entropy": -91.67915344238281, + "objective/kl": 37.619895935058594, + "objective/non_score_reward": -1.8809947967529297, + "objective/rlhf_reward": 14.22056770324707, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009494669502601027, + "policy/clipfrac_avg": 0.007099117152392864, + "policy/entropy_avg": 0.17319107055664062, + "step": 908, + "val/clipfrac_avg": 0.07242873311042786, + "val/num_eos_tokens": 437, + "val/ratio": 0.9883796572685242, + "val/ratio_var": 7.36714864615351e-05 + }, + { + "episode": 14544, + "epoch": 0.7345454545454545, + "eps": 0, + "loss/policy_avg": 0.15317559242248535, + "loss/value_avg": 0.7379442453384399, + "lr": 1.8202960607577246e-06, + "objective/entropy": -92.03844451904297, + "objective/kl": 36.91444396972656, + "objective/non_score_reward": -1.8457221984863281, + "objective/rlhf_reward": 13.802715301513672, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0009770962642505765, + "policy/clipfrac_avg": 0.005569060333073139, + "policy/entropy_avg": 0.18958091735839844, + "step": 909, + "val/clipfrac_avg": 0.04051949828863144, + "val/num_eos_tokens": 56, + "val/ratio": 0.988923192024231, + "val/ratio_var": 6.142368511063978e-05 + }, + { + "episode": 14560, + "epoch": 0.7353535353535353, + "eps": 0, + "loss/policy_avg": 0.1602431833744049, + "loss/value_avg": 0.3025469183921814, + "lr": 1.809999230334958e-06, + "objective/entropy": -111.80148315429688, + "objective/kl": 38.83802032470703, + "objective/non_score_reward": -1.9419012069702148, + "objective/rlhf_reward": 14.401848793029785, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000904000480659306, + "policy/clipfrac_avg": 0.005316127557307482, + "policy/entropy_avg": 0.17273902893066406, + "step": 910, + "val/clipfrac_avg": 0.00018382353300694376, + "val/num_eos_tokens": 342, + "val/ratio": 0.9891130924224854, + "val/ratio_var": 7.234779332065955e-05 + }, + { + "episode": 14576, + "epoch": 0.7361616161616161, + "eps": 0, + "loss/policy_avg": 0.09882547706365585, + "loss/value_avg": 0.6121327877044678, + "lr": 1.7997251653303249e-06, + "objective/entropy": -98.02986145019531, + "objective/kl": 44.55670166015625, + "objective/non_score_reward": -2.227834939956665, + "objective/rlhf_reward": 13.092477798461914, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009336632210761309, + "policy/clipfrac_avg": 0.004822542425245047, + "policy/entropy_avg": 0.17920780181884766, + "step": 911, + "val/clipfrac_avg": 0.0050772735849022865, + "val/num_eos_tokens": 426, + "val/ratio": 0.9875223636627197, + "val/ratio_var": 6.380004924722016e-05 + }, + { + "episode": 14592, + "epoch": 0.7369696969696969, + "eps": 0, + "loss/policy_avg": -0.15951846539974213, + "loss/value_avg": 0.5648845434188843, + "lr": 1.7894739390646227e-06, + "objective/entropy": -57.16059875488281, + "objective/kl": 36.05466079711914, + "objective/non_score_reward": -1.802733063697815, + "objective/rlhf_reward": 13.119142532348633, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009670511353760958, + "policy/clipfrac_avg": 0.00586615689098835, + "policy/entropy_avg": 0.1721019744873047, + "step": 912, + "val/clipfrac_avg": 0.0008223684271797538, + "val/num_eos_tokens": 45, + "val/ratio": 0.9892112016677856, + "val/ratio_var": 4.9591733841225505e-05 + }, + { + "episode": 14608, + "epoch": 0.7377777777777778, + "eps": 0, + "loss/policy_avg": 0.1940194070339203, + "loss/value_avg": 0.3815060555934906, + "lr": 1.7792456246956535e-06, + "objective/entropy": -146.6658935546875, + "objective/kl": 46.18011474609375, + "objective/non_score_reward": -2.3090057373046875, + "objective/rlhf_reward": 14.511306762695312, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009307245491072536, + "policy/clipfrac_avg": 0.006143895909190178, + "policy/entropy_avg": 0.18556424975395203, + "step": 913, + "val/clipfrac_avg": 0.0180903859436512, + "val/num_eos_tokens": 698, + "val/ratio": 0.9874189496040344, + "val/ratio_var": 0.00011522904242156073 + }, + { + "episode": 14624, + "epoch": 0.7385858585858586, + "eps": 0, + "loss/policy_avg": -0.02095976285636425, + "loss/value_avg": 0.7224372625350952, + "lr": 1.7690402952177117e-06, + "objective/entropy": -84.84890747070312, + "objective/kl": 35.78252410888672, + "objective/non_score_reward": -1.7891263961791992, + "objective/rlhf_reward": 13.2655611038208, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009321504039689898, + "policy/clipfrac_avg": 0.005007730796933174, + "policy/entropy_avg": 0.18702125549316406, + "step": 914, + "val/clipfrac_avg": 0.023768942803144455, + "val/num_eos_tokens": 403, + "val/ratio": 0.9874284267425537, + "val/ratio_var": 8.100390550680459e-05 + }, + { + "episode": 14640, + "epoch": 0.7393939393939394, + "eps": 0, + "loss/policy_avg": -0.3134731650352478, + "loss/value_avg": 0.5765621066093445, + "lr": 1.7588580234610592e-06, + "objective/entropy": -63.22113037109375, + "objective/kl": 38.5015869140625, + "objective/non_score_reward": -1.925079345703125, + "objective/rlhf_reward": 13.918670654296875, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008501409320160747, + "policy/clipfrac_avg": 0.004706401377916336, + "policy/entropy_avg": 0.1600513458251953, + "step": 915, + "val/clipfrac_avg": 0.06931675970554352, + "val/num_eos_tokens": 171, + "val/ratio": 0.9882992506027222, + "val/ratio_var": 6.202542135724798e-05 + }, + { + "episode": 14656, + "epoch": 0.7402020202020202, + "eps": 0, + "loss/policy_avg": -0.2850199043750763, + "loss/value_avg": 0.8679437637329102, + "lr": 1.7486988820914063e-06, + "objective/entropy": -43.271400451660156, + "objective/kl": 42.955291748046875, + "objective/non_score_reward": -2.1477646827697754, + "objective/rlhf_reward": 13.875673294067383, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009495788253843784, + "policy/clipfrac_avg": 0.005583056248724461, + "policy/entropy_avg": 0.17831039428710938, + "step": 916, + "val/clipfrac_avg": 0.06379853188991547, + "val/num_eos_tokens": 103, + "val/ratio": 0.9876798391342163, + "val/ratio_var": 4.0028760849963874e-05 + }, + { + "episode": 14672, + "epoch": 0.741010101010101, + "eps": 0, + "loss/policy_avg": 0.21622401475906372, + "loss/value_avg": 0.22584834694862366, + "lr": 1.7385629436093958e-06, + "objective/entropy": -79.67193603515625, + "objective/kl": 44.639923095703125, + "objective/non_score_reward": -2.2319962978363037, + "objective/rlhf_reward": 13.463315963745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008941494161263108, + "policy/clipfrac_avg": 0.006674608215689659, + "policy/entropy_avg": 0.1719951629638672, + "step": 917, + "val/clipfrac_avg": 0.029584797099232674, + "val/num_eos_tokens": 50, + "val/ratio": 0.9886019825935364, + "val/ratio_var": 4.6610963181592524e-05 + }, + { + "episode": 14688, + "epoch": 0.7418181818181818, + "eps": 0, + "loss/policy_avg": -0.0006486065685749054, + "loss/value_avg": 0.2962718605995178, + "lr": 1.728450280350079e-06, + "objective/entropy": -68.00552368164062, + "objective/kl": 38.6593132019043, + "objective/non_score_reward": -1.9329655170440674, + "objective/rlhf_reward": 13.887346267700195, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008251155959442258, + "policy/clipfrac_avg": 0.005509347654879093, + "policy/entropy_avg": 0.1643085479736328, + "step": 918, + "val/clipfrac_avg": 0.003826581174507737, + "val/num_eos_tokens": 44, + "val/ratio": 0.9888970851898193, + "val/ratio_var": 3.0508910640492104e-05 + }, + { + "episode": 14704, + "epoch": 0.7426262626262626, + "eps": 0, + "loss/policy_avg": -0.3462671935558319, + "loss/value_avg": 0.7152901887893677, + "lr": 1.7183609644824096e-06, + "objective/entropy": -93.26936340332031, + "objective/kl": 40.42371368408203, + "objective/non_score_reward": -2.021185874938965, + "objective/rlhf_reward": 15.166314125061035, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009743100963532925, + "policy/clipfrac_avg": 0.0028937258757650852, + "policy/entropy_avg": 0.18361568450927734, + "step": 919, + "val/clipfrac_avg": 0.049027521163225174, + "val/num_eos_tokens": 136, + "val/ratio": 0.9877507090568542, + "val/ratio_var": 5.374600732466206e-05 + }, + { + "episode": 14720, + "epoch": 0.7434343434343434, + "eps": 0, + "loss/policy_avg": 0.1274300217628479, + "loss/value_avg": 1.230905294418335, + "lr": 1.708295068008718e-06, + "objective/entropy": -81.48149108886719, + "objective/kl": 36.17670440673828, + "objective/non_score_reward": -1.8088352680206299, + "objective/rlhf_reward": 12.589601516723633, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008495738729834557, + "policy/clipfrac_avg": 0.0038070818409323692, + "policy/entropy_avg": 0.16942787170410156, + "step": 920, + "val/clipfrac_avg": 0.007427365053445101, + "val/num_eos_tokens": 270, + "val/ratio": 0.9902092218399048, + "val/ratio_var": 5.2669711294583976e-05 + }, + { + "episode": 14736, + "epoch": 0.7442424242424243, + "eps": 0, + "loss/policy_avg": -0.257866770029068, + "loss/value_avg": 0.5942734479904175, + "lr": 1.6982526627642043e-06, + "objective/entropy": -74.04690551757812, + "objective/kl": 42.419307708740234, + "objective/non_score_reward": -2.1209654808044434, + "objective/rlhf_reward": 13.511846542358398, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000907333567738533, + "policy/clipfrac_avg": 0.004857117310166359, + "policy/entropy_avg": 0.18361282348632812, + "step": 921, + "val/clipfrac_avg": 0.043217435479164124, + "val/num_eos_tokens": 56, + "val/ratio": 0.9877738356590271, + "val/ratio_var": 6.433074304368347e-05 + }, + { + "episode": 14752, + "epoch": 0.7450505050505051, + "eps": 0, + "loss/policy_avg": 0.12189102917909622, + "loss/value_avg": 0.41360408067703247, + "lr": 1.6882338204164212e-06, + "objective/entropy": -54.975791931152344, + "objective/kl": 35.72700881958008, + "objective/non_score_reward": -1.7863504886627197, + "objective/rlhf_reward": 13.39333724975586, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010183433769270778, + "policy/clipfrac_avg": 0.003768398892134428, + "policy/entropy_avg": 0.1844940185546875, + "step": 922, + "val/clipfrac_avg": 0.0008610248914919794, + "val/num_eos_tokens": 362, + "val/ratio": 0.9872738122940063, + "val/ratio_var": 3.0919560231268406e-05 + }, + { + "episode": 14768, + "epoch": 0.7458585858585859, + "eps": 0, + "loss/policy_avg": -0.07059650868177414, + "loss/value_avg": 0.6463966369628906, + "lr": 1.67823861246477e-06, + "objective/entropy": -15.746962547302246, + "objective/kl": 40.59729766845703, + "objective/non_score_reward": -2.029865026473999, + "objective/rlhf_reward": 12.388103485107422, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010770086664706469, + "policy/clipfrac_avg": 0.005413630045950413, + "policy/entropy_avg": 0.21207809448242188, + "step": 923, + "val/clipfrac_avg": 0.00856552179902792, + "val/num_eos_tokens": 175, + "val/ratio": 0.9861268401145935, + "val/ratio_var": 5.63512985536363e-05 + }, + { + "episode": 14784, + "epoch": 0.7466666666666667, + "eps": 0, + "loss/policy_avg": 0.06915351748466492, + "loss/value_avg": 1.4651566743850708, + "lr": 1.6682671102399806e-06, + "objective/entropy": -118.91422271728516, + "objective/kl": 34.25519943237305, + "objective/non_score_reward": -1.7127599716186523, + "objective/rlhf_reward": 12.224739074707031, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0007098158821463585, + "policy/clipfrac_avg": 0.0022262982092797756, + "policy/entropy_avg": 0.1386256217956543, + "step": 924, + "val/clipfrac_avg": 0.06050518900156021, + "val/num_eos_tokens": 270, + "val/ratio": 0.9912421703338623, + "val/ratio_var": 4.530951628112234e-05 + }, + { + "episode": 14800, + "epoch": 0.7474747474747475, + "eps": 0, + "loss/policy_avg": 0.18248242139816284, + "loss/value_avg": 1.6025516986846924, + "lr": 1.6583193849036095e-06, + "objective/entropy": -81.5009765625, + "objective/kl": 38.95854949951172, + "objective/non_score_reward": -1.947927474975586, + "objective/rlhf_reward": 15.450510025024414, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009277502540498972, + "policy/clipfrac_avg": 0.004952313844114542, + "policy/entropy_avg": 0.16534215211868286, + "step": 925, + "val/clipfrac_avg": 0.016582708805799484, + "val/num_eos_tokens": 323, + "val/ratio": 0.9883300065994263, + "val/ratio_var": 5.890153624932282e-05 + }, + { + "episode": 14816, + "epoch": 0.7482828282828283, + "eps": 0, + "loss/policy_avg": -0.16795751452445984, + "loss/value_avg": 0.3992899954319, + "lr": 1.6483955074475278e-06, + "objective/entropy": -46.540870666503906, + "objective/kl": 35.18474578857422, + "objective/non_score_reward": -1.75923752784729, + "objective/rlhf_reward": 13.225137710571289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009727986180223525, + "policy/clipfrac_avg": 0.0023795529268682003, + "policy/entropy_avg": 0.21690034866333008, + "step": 926, + "val/clipfrac_avg": 0.0444885715842247, + "val/num_eos_tokens": 163, + "val/ratio": 0.9869061708450317, + "val/ratio_var": 6.481046148110181e-05 + }, + { + "episode": 14832, + "epoch": 0.7490909090909091, + "eps": 0, + "loss/policy_avg": -0.3186694383621216, + "loss/value_avg": 0.933090090751648, + "lr": 1.6384955486934157e-06, + "objective/entropy": -73.27378845214844, + "objective/kl": 33.047142028808594, + "objective/non_score_reward": -1.6523573398590088, + "objective/rlhf_reward": 13.33201789855957, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008205980993807316, + "policy/clipfrac_avg": 0.0039194016717374325, + "policy/entropy_avg": 0.17404580116271973, + "step": 927, + "val/clipfrac_avg": 0.07463380694389343, + "val/num_eos_tokens": 165, + "val/ratio": 0.9890973567962646, + "val/ratio_var": 8.332987636094913e-05 + }, + { + "episode": 14848, + "epoch": 0.74989898989899, + "eps": 0, + "loss/policy_avg": -0.018651030957698822, + "loss/value_avg": 0.23730480670928955, + "lr": 1.6286195792922655e-06, + "objective/entropy": -108.95191192626953, + "objective/kl": 41.2000846862793, + "objective/non_score_reward": -2.060004472732544, + "objective/rlhf_reward": 14.135308265686035, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007390918326564133, + "policy/clipfrac_avg": 0.0021800659596920013, + "policy/entropy_avg": 0.15365886688232422, + "step": 928, + "val/clipfrac_avg": 0.02866453304886818, + "val/num_eos_tokens": 497, + "val/ratio": 0.989920973777771, + "val/ratio_var": 5.118203625897877e-05 + }, + { + "episode": 14864, + "epoch": 0.7507070707070707, + "eps": 0, + "loss/policy_avg": 0.11356082558631897, + "loss/value_avg": 0.5069226622581482, + "lr": 1.6187676697238564e-06, + "objective/entropy": -53.78815841674805, + "objective/kl": 47.88815689086914, + "objective/non_score_reward": -2.3944079875946045, + "objective/rlhf_reward": 12.574342727661133, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008938192040659487, + "policy/clipfrac_avg": 0.003149396274238825, + "policy/entropy_avg": 0.181840181350708, + "step": 929, + "val/clipfrac_avg": 0.0172991082072258, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889012575149536, + "val/ratio_var": 3.912192551069893e-05 + }, + { + "episode": 14880, + "epoch": 0.7515151515151515, + "eps": 0, + "loss/policy_avg": -0.07865975797176361, + "loss/value_avg": 0.41433531045913696, + "lr": 1.6089398902962767e-06, + "objective/entropy": -104.1807861328125, + "objective/kl": 37.49846649169922, + "objective/non_score_reward": -1.8749234676361084, + "objective/rlhf_reward": 15.953201293945312, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009115774882957339, + "policy/clipfrac_avg": 0.005451949778944254, + "policy/entropy_avg": 0.1745290756225586, + "step": 930, + "val/clipfrac_avg": 0.058040205389261246, + "val/num_eos_tokens": 27, + "val/ratio": 0.9887331128120422, + "val/ratio_var": 6.705939449602738e-05 + }, + { + "episode": 14896, + "epoch": 0.7523232323232323, + "eps": 0, + "loss/policy_avg": -0.035402484238147736, + "loss/value_avg": 0.5358971357345581, + "lr": 1.5991363111454023e-06, + "objective/entropy": -97.50758361816406, + "objective/kl": 42.68102264404297, + "objective/non_score_reward": -2.1340510845184326, + "objective/rlhf_reward": 13.498762130737305, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008075715741142631, + "policy/clipfrac_avg": 0.011311197653412819, + "policy/entropy_avg": 0.1630105972290039, + "step": 931, + "val/clipfrac_avg": 0.003885046113282442, + "val/num_eos_tokens": 302, + "val/ratio": 0.9890309572219849, + "val/ratio_var": 5.2104482165304944e-05 + }, + { + "episode": 14912, + "epoch": 0.7531313131313131, + "eps": 0, + "loss/policy_avg": -0.30045461654663086, + "loss/value_avg": 0.872713565826416, + "lr": 1.5893570022344124e-06, + "objective/entropy": -78.21574401855469, + "objective/kl": 32.467742919921875, + "objective/non_score_reward": -1.623387098312378, + "objective/rlhf_reward": 14.372706413269043, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008647768991068006, + "policy/clipfrac_avg": 0.005656212102621794, + "policy/entropy_avg": 0.17721939086914062, + "step": 932, + "val/clipfrac_avg": 0.0015919646248221397, + "val/num_eos_tokens": 12, + "val/ratio": 0.9885766506195068, + "val/ratio_var": 6.424664752557874e-05 + }, + { + "episode": 14928, + "epoch": 0.7539393939393939, + "eps": 0, + "loss/policy_avg": -0.012097490951418877, + "loss/value_avg": 0.7004050016403198, + "lr": 1.5796020333532696e-06, + "objective/entropy": -61.095802307128906, + "objective/kl": 36.504150390625, + "objective/non_score_reward": -1.8252074718475342, + "objective/rlhf_reward": 12.393542289733887, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008130621863529086, + "policy/clipfrac_avg": 0.005108194425702095, + "policy/entropy_avg": 0.16521072387695312, + "step": 933, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 182, + "val/ratio": 0.9887269735336304, + "val/ratio_var": 6.136602314654738e-05 + }, + { + "episode": 14944, + "epoch": 0.7547474747474747, + "eps": 0, + "loss/policy_avg": 0.03149443119764328, + "loss/value_avg": 0.3308320641517639, + "lr": 1.5698714741182459e-06, + "objective/entropy": -110.07270812988281, + "objective/kl": 34.973388671875, + "objective/non_score_reward": -1.7486693859100342, + "objective/rlhf_reward": 14.548205375671387, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00091616774443537, + "policy/clipfrac_avg": 0.006772395223379135, + "policy/entropy_avg": 0.17972993850708008, + "step": 934, + "val/clipfrac_avg": 0.0015980113530531526, + "val/num_eos_tokens": 350, + "val/ratio": 0.9878275394439697, + "val/ratio_var": 8.512718341080472e-05 + }, + { + "episode": 14960, + "epoch": 0.7555555555555555, + "eps": 0, + "loss/policy_avg": -0.03353989124298096, + "loss/value_avg": 0.4103987216949463, + "lr": 1.5601653939714073e-06, + "objective/entropy": -25.034250259399414, + "objective/kl": 39.53004837036133, + "objective/non_score_reward": -1.976502537727356, + "objective/rlhf_reward": 14.281310081481934, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102006527595222, + "policy/clipfrac_avg": 0.00497299712151289, + "policy/entropy_avg": 0.19387245178222656, + "step": 935, + "val/clipfrac_avg": 0.0242628026753664, + "val/num_eos_tokens": 73, + "val/ratio": 0.9875220060348511, + "val/ratio_var": 6.656184268649668e-05 + }, + { + "episode": 14976, + "epoch": 0.7563636363636363, + "eps": 0, + "loss/policy_avg": -0.1474141776561737, + "loss/value_avg": 0.527273416519165, + "lr": 1.5504838621801272e-06, + "objective/entropy": -60.77607345581055, + "objective/kl": 37.98503494262695, + "objective/non_score_reward": -1.8992516994476318, + "objective/rlhf_reward": 13.171060562133789, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007948282873257995, + "policy/clipfrac_avg": 0.002496684668585658, + "policy/entropy_avg": 0.1510624885559082, + "step": 936, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 3.4012387914117426e-05 + }, + { + "episode": 14992, + "epoch": 0.7571717171717172, + "eps": 0, + "loss/policy_avg": 0.1368754655122757, + "loss/value_avg": 0.6238852143287659, + "lr": 1.5408269478365883e-06, + "objective/entropy": -60.082557678222656, + "objective/kl": 41.95027160644531, + "objective/non_score_reward": -2.0975136756896973, + "objective/rlhf_reward": 12.652486801147461, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009200765052810311, + "policy/clipfrac_avg": 0.0059701185673475266, + "policy/entropy_avg": 0.17743873596191406, + "step": 937, + "val/clipfrac_avg": 0.012415202334523201, + "val/num_eos_tokens": 54, + "val/ratio": 0.9879815578460693, + "val/ratio_var": 4.387104490888305e-05 + }, + { + "episode": 15008, + "epoch": 0.757979797979798, + "eps": 0, + "loss/policy_avg": -0.17327642440795898, + "loss/value_avg": 0.538246750831604, + "lr": 1.5311947198572918e-06, + "objective/entropy": -112.668701171875, + "objective/kl": 35.57097625732422, + "objective/non_score_reward": -1.7785489559173584, + "objective/rlhf_reward": 13.416763305664062, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008795931353233755, + "policy/clipfrac_avg": 0.005144854076206684, + "policy/entropy_avg": 0.16583538055419922, + "step": 938, + "val/clipfrac_avg": 0.047972679138183594, + "val/num_eos_tokens": 136, + "val/ratio": 0.9902178645133972, + "val/ratio_var": 5.2144809160381556e-05 + }, + { + "episode": 15024, + "epoch": 0.7587878787878788, + "eps": 0, + "loss/policy_avg": 0.5920745134353638, + "loss/value_avg": 1.1476694345474243, + "lr": 1.5215872469825682e-06, + "objective/entropy": -72.79998779296875, + "objective/kl": 35.88062286376953, + "objective/non_score_reward": -1.7940311431884766, + "objective/rlhf_reward": 14.46378231048584, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008520716801285744, + "policy/clipfrac_avg": 0.0030313697643578053, + "policy/entropy_avg": 0.19948351383209229, + "step": 939, + "val/clipfrac_avg": 0.021162573248147964, + "val/num_eos_tokens": 210, + "val/ratio": 0.9882235527038574, + "val/ratio_var": 6.201990618137643e-05 + }, + { + "episode": 15040, + "epoch": 0.7595959595959596, + "eps": 0, + "loss/policy_avg": -0.15639176964759827, + "loss/value_avg": 0.8813156485557556, + "lr": 1.5120045977760784e-06, + "objective/entropy": -91.86679077148438, + "objective/kl": 42.35734176635742, + "objective/non_score_reward": -2.1178669929504395, + "objective/rlhf_reward": 15.264945983886719, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0010741858277469873, + "policy/clipfrac_avg": 0.00790454912930727, + "policy/entropy_avg": 0.19293498992919922, + "step": 940, + "val/clipfrac_avg": 0.015625, + "val/num_eos_tokens": 9, + "val/ratio": 0.986987829208374, + "val/ratio_var": 6.426944310078397e-05 + }, + { + "episode": 15056, + "epoch": 0.7604040404040404, + "eps": 0, + "loss/policy_avg": 0.2165047824382782, + "loss/value_avg": 0.34554335474967957, + "lr": 1.502446840624331e-06, + "objective/entropy": -62.31670379638672, + "objective/kl": 31.252155303955078, + "objective/non_score_reward": -1.562608003616333, + "objective/rlhf_reward": 14.156142234802246, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007184218848124146, + "policy/clipfrac_avg": 0.0035393601283431053, + "policy/entropy_avg": 0.14359378814697266, + "step": 941, + "val/clipfrac_avg": 0.013209432363510132, + "val/num_eos_tokens": 36, + "val/ratio": 0.9896091222763062, + "val/ratio_var": 5.599396536126733e-05 + }, + { + "episode": 15072, + "epoch": 0.7612121212121212, + "eps": 0, + "loss/policy_avg": -0.030075984075665474, + "loss/value_avg": 0.45760178565979004, + "lr": 1.4929140437361916e-06, + "objective/entropy": -109.5699691772461, + "objective/kl": 38.075103759765625, + "objective/non_score_reward": -1.9037551879882812, + "objective/rlhf_reward": 14.713432312011719, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010412151459604502, + "policy/clipfrac_avg": 0.009930627420544624, + "policy/entropy_avg": 0.18667316436767578, + "step": 942, + "val/clipfrac_avg": 0.014139890670776367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9863598346710205, + "val/ratio_var": 0.00010634912905516103 + }, + { + "episode": 15088, + "epoch": 0.762020202020202, + "eps": 0, + "loss/policy_avg": 0.12722979485988617, + "loss/value_avg": 0.5696362853050232, + "lr": 1.4834062751424018e-06, + "objective/entropy": -121.374267578125, + "objective/kl": 34.23678207397461, + "objective/non_score_reward": -1.711839199066162, + "objective/rlhf_reward": 14.038161277770996, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007155380444601178, + "policy/clipfrac_avg": 0.00446352269500494, + "policy/entropy_avg": 0.13922500610351562, + "step": 943, + "val/clipfrac_avg": 0.001972501166164875, + "val/num_eos_tokens": 493, + "val/ratio": 0.9906710386276245, + "val/ratio_var": 6.088761438149959e-05 + }, + { + "episode": 15104, + "epoch": 0.7628282828282829, + "eps": 0, + "loss/policy_avg": 0.24459479749202728, + "loss/value_avg": 0.997427225112915, + "lr": 1.4739236026950838e-06, + "objective/entropy": -117.67542266845703, + "objective/kl": 42.60675811767578, + "objective/non_score_reward": -2.130337953567505, + "objective/rlhf_reward": 13.854036331176758, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007362811593338847, + "policy/clipfrac_avg": 0.009282913990318775, + "policy/entropy_avg": 0.15163874626159668, + "step": 944, + "val/clipfrac_avg": 0.019384628161787987, + "val/num_eos_tokens": 404, + "val/ratio": 0.9907106161117554, + "val/ratio_var": 5.1745548262260854e-05 + }, + { + "episode": 15120, + "epoch": 0.7636363636363637, + "eps": 0, + "loss/policy_avg": 0.019024059176445007, + "loss/value_avg": 0.2505372166633606, + "lr": 1.4644660940672628e-06, + "objective/entropy": -72.49942779541016, + "objective/kl": 36.86358642578125, + "objective/non_score_reward": -1.8431792259216309, + "objective/rlhf_reward": 14.227133750915527, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009655736503191292, + "policy/clipfrac_avg": 0.0035610971972346306, + "policy/entropy_avg": 0.19275665283203125, + "step": 945, + "val/clipfrac_avg": 0.0008617835119366646, + "val/num_eos_tokens": 33, + "val/ratio": 0.9873601198196411, + "val/ratio_var": 7.99532572273165e-05 + }, + { + "episode": 15136, + "epoch": 0.7644444444444445, + "eps": 0, + "loss/policy_avg": -0.24745053052902222, + "loss/value_avg": 0.4052670896053314, + "lr": 1.455033816752383e-06, + "objective/entropy": -106.38670349121094, + "objective/kl": 30.419466018676758, + "objective/non_score_reward": -1.5209734439849854, + "objective/rlhf_reward": 15.338401794433594, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009057885035872459, + "policy/clipfrac_avg": 0.009060706943273544, + "policy/entropy_avg": 0.161224365234375, + "step": 946, + "val/clipfrac_avg": 0.038856737315654755, + "val/num_eos_tokens": 409, + "val/ratio": 0.9891325235366821, + "val/ratio_var": 9.022479935083538e-05 + }, + { + "episode": 15152, + "epoch": 0.7652525252525253, + "eps": 0, + "loss/policy_avg": -0.051016077399253845, + "loss/value_avg": 0.5337303876876831, + "lr": 1.4456268380638262e-06, + "objective/entropy": -104.14939880371094, + "objective/kl": 36.21892547607422, + "objective/non_score_reward": -1.8109462261199951, + "objective/rlhf_reward": 15.282804489135742, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009193423902615905, + "policy/clipfrac_avg": 0.010075472295284271, + "policy/entropy_avg": 0.16665363311767578, + "step": 947, + "val/clipfrac_avg": 0.006654549390077591, + "val/num_eos_tokens": 424, + "val/ratio": 0.9888213276863098, + "val/ratio_var": 8.222387987188995e-05 + }, + { + "episode": 15168, + "epoch": 0.7660606060606061, + "eps": 0, + "loss/policy_avg": 0.5708428025245667, + "loss/value_avg": 0.9156904220581055, + "lr": 1.4362452251344283e-06, + "objective/entropy": -107.8341064453125, + "objective/kl": 37.6370849609375, + "objective/non_score_reward": -1.8818542957305908, + "objective/rlhf_reward": 11.461895942687988, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008800949435681105, + "policy/clipfrac_avg": 0.018943525850772858, + "policy/entropy_avg": 0.18418407440185547, + "step": 948, + "val/clipfrac_avg": 0.0006793478387407959, + "val/num_eos_tokens": 297, + "val/ratio": 0.9880671501159668, + "val/ratio_var": 7.767622446408495e-05 + }, + { + "episode": 15184, + "epoch": 0.7668686868686869, + "eps": 0, + "loss/policy_avg": -0.028068287298083305, + "loss/value_avg": 0.6219231486320496, + "lr": 1.4268890449160034e-06, + "objective/entropy": -11.566868782043457, + "objective/kl": 34.966495513916016, + "objective/non_score_reward": -1.7483247518539429, + "objective/rlhf_reward": 14.603238105773926, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0011735728476196527, + "policy/clipfrac_avg": 0.004941962193697691, + "policy/entropy_avg": 0.20639419555664062, + "step": 949, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 32, + "val/ratio": 0.9847333431243896, + "val/ratio_var": 3.736758299055509e-05 + }, + { + "episode": 15200, + "epoch": 0.7676767676767676, + "eps": 0, + "loss/policy_avg": -0.15076535940170288, + "loss/value_avg": 0.2775721549987793, + "lr": 1.4175583641788698e-06, + "objective/entropy": -80.4765625, + "objective/kl": 31.678754806518555, + "objective/non_score_reward": -1.583937644958496, + "objective/rlhf_reward": 13.853562355041504, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0006830325582996011, + "policy/clipfrac_avg": 0.00417131744325161, + "policy/entropy_avg": 0.1293621063232422, + "step": 950, + "val/clipfrac_avg": 0.0393822118639946, + "val/num_eos_tokens": 257, + "val/ratio": 0.9911376237869263, + "val/ratio_var": 2.9909415388829075e-05 + }, + { + "episode": 15216, + "epoch": 0.7684848484848484, + "eps": 0, + "loss/policy_avg": -0.04640840366482735, + "loss/value_avg": 0.3922756314277649, + "lr": 1.4082532495113627e-06, + "objective/entropy": -84.18096160888672, + "objective/kl": 38.69972229003906, + "objective/non_score_reward": -1.9349862337112427, + "objective/rlhf_reward": 15.416576385498047, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.000798642635345459, + "policy/clipfrac_avg": 0.0038171219639480114, + "policy/entropy_avg": 0.1776995062828064, + "step": 951, + "val/clipfrac_avg": 0.0032731681130826473, + "val/num_eos_tokens": 326, + "val/ratio": 0.9889781475067139, + "val/ratio_var": 5.715083170798607e-05 + }, + { + "episode": 15232, + "epoch": 0.7692929292929293, + "eps": 0, + "loss/policy_avg": -0.12606917321681976, + "loss/value_avg": 0.3175104856491089, + "lr": 1.3989737673193682e-06, + "objective/entropy": -60.43904113769531, + "objective/kl": 39.91594314575195, + "objective/non_score_reward": -1.9957971572875977, + "objective/rlhf_reward": 14.262016296386719, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009779632091522217, + "policy/clipfrac_avg": 0.004585625603795052, + "policy/entropy_avg": 0.2008209228515625, + "step": 952, + "val/clipfrac_avg": 0.030662208795547485, + "val/num_eos_tokens": 422, + "val/ratio": 0.9862244129180908, + "val/ratio_var": 8.592133963247761e-05 + }, + { + "episode": 15248, + "epoch": 0.7701010101010101, + "eps": 0, + "loss/policy_avg": 0.08768834173679352, + "loss/value_avg": 0.327705055475235, + "lr": 1.389719983825844e-06, + "objective/entropy": -67.14826965332031, + "objective/kl": 46.73610305786133, + "objective/non_score_reward": -2.3368051052093506, + "objective/rlhf_reward": 14.29600715637207, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011253968114033341, + "policy/clipfrac_avg": 0.006955197546631098, + "policy/entropy_avg": 0.20795369148254395, + "step": 953, + "val/clipfrac_avg": 8.926542068365961e-05, + "val/num_eos_tokens": 421, + "val/ratio": 0.9854748249053955, + "val/ratio_var": 6.634320016019046e-05 + }, + { + "episode": 15264, + "epoch": 0.7709090909090909, + "eps": 0, + "loss/policy_avg": 0.17172515392303467, + "loss/value_avg": 0.5182530283927917, + "lr": 1.3804919650703551e-06, + "objective/entropy": -52.92948913574219, + "objective/kl": 39.47554016113281, + "objective/non_score_reward": -1.9737770557403564, + "objective/rlhf_reward": 15.323097229003906, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009001094149425626, + "policy/clipfrac_avg": 0.005291195120662451, + "policy/entropy_avg": 0.18181610107421875, + "step": 954, + "val/clipfrac_avg": 0.006615057587623596, + "val/num_eos_tokens": 490, + "val/ratio": 0.9876183271408081, + "val/ratio_var": 5.796940240543336e-05 + }, + { + "episode": 15280, + "epoch": 0.7717171717171717, + "eps": 0, + "loss/policy_avg": 0.3471048176288605, + "loss/value_avg": 0.7186154723167419, + "lr": 1.3712897769085903e-06, + "objective/entropy": -93.25178527832031, + "objective/kl": 41.423980712890625, + "objective/non_score_reward": -2.0711989402770996, + "objective/rlhf_reward": 13.155363082885742, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009469924843870103, + "policy/clipfrac_avg": 0.004164956044405699, + "policy/entropy_avg": 0.18822860717773438, + "step": 955, + "val/clipfrac_avg": 0.04515676945447922, + "val/num_eos_tokens": 611, + "val/ratio": 0.9877511262893677, + "val/ratio_var": 7.300172001123428e-05 + }, + { + "episode": 15296, + "epoch": 0.7725252525252525, + "eps": 0, + "loss/policy_avg": 0.24007092416286469, + "loss/value_avg": 0.4516885280609131, + "lr": 1.3621134850119012e-06, + "objective/entropy": -112.50715637207031, + "objective/kl": 31.20224380493164, + "objective/non_score_reward": -1.5601121187210083, + "objective/rlhf_reward": 13.799263000488281, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007952903397381306, + "policy/clipfrac_avg": 0.008523119613528252, + "policy/entropy_avg": 0.1457352638244629, + "step": 956, + "val/clipfrac_avg": 0.06015515699982643, + "val/num_eos_tokens": 383, + "val/ratio": 0.9895913600921631, + "val/ratio_var": 7.848602399462834e-05 + }, + { + "episode": 15312, + "epoch": 0.7733333333333333, + "eps": 0, + "loss/policy_avg": -0.03721753507852554, + "loss/value_avg": 0.241817444562912, + "lr": 1.3529631548668298e-06, + "objective/entropy": -82.6330337524414, + "objective/kl": 44.714515686035156, + "objective/non_score_reward": -2.2357258796691895, + "objective/rlhf_reward": 13.818962097167969, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0006799673428758979, + "policy/clipfrac_avg": 0.003063515294343233, + "policy/entropy_avg": 0.1481151580810547, + "step": 957, + "val/clipfrac_avg": 9.245562250725925e-05, + "val/num_eos_tokens": 341, + "val/ratio": 0.9905509948730469, + "val/ratio_var": 4.010691918665543e-05 + }, + { + "episode": 15328, + "epoch": 0.7741414141414141, + "eps": 0, + "loss/policy_avg": -0.1448572874069214, + "loss/value_avg": 0.6361722946166992, + "lr": 1.3438388517746431e-06, + "objective/entropy": -75.73651885986328, + "objective/kl": 32.26376724243164, + "objective/non_score_reward": -1.6131882667541504, + "objective/rlhf_reward": 13.550874710083008, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010009508114308119, + "policy/clipfrac_avg": 0.004912985023111105, + "policy/entropy_avg": 0.190643310546875, + "step": 958, + "val/clipfrac_avg": 0.008336715400218964, + "val/num_eos_tokens": 329, + "val/ratio": 0.9864659905433655, + "val/ratio_var": 9.685769327916205e-05 + }, + { + "episode": 15344, + "epoch": 0.7749494949494949, + "eps": 0, + "loss/policy_avg": 0.11857907474040985, + "loss/value_avg": 0.8868049383163452, + "lr": 1.3347406408508695e-06, + "objective/entropy": -90.25643920898438, + "objective/kl": 39.02738952636719, + "objective/non_score_reward": -1.9513695240020752, + "objective/rlhf_reward": 14.954880714416504, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009018229902721941, + "policy/clipfrac_avg": 0.004778256639838219, + "policy/entropy_avg": 0.1796565055847168, + "step": 959, + "val/clipfrac_avg": 0.02706507220864296, + "val/num_eos_tokens": 289, + "val/ratio": 0.9882634878158569, + "val/ratio_var": 9.323142148787156e-05 + }, + { + "episode": 15360, + "epoch": 0.7757575757575758, + "eps": 0, + "loss/policy_avg": -0.22503328323364258, + "loss/value_avg": 0.4810031056404114, + "lr": 1.3256685870248227e-06, + "objective/entropy": -101.29171752929688, + "objective/kl": 37.34764099121094, + "objective/non_score_reward": -1.8673819303512573, + "objective/rlhf_reward": 14.32011890411377, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008543966105207801, + "policy/clipfrac_avg": 0.0047050644643604755, + "policy/entropy_avg": 0.17150402069091797, + "step": 960, + "val/clipfrac_avg": 0.0005405173869803548, + "val/num_eos_tokens": 130, + "val/ratio": 0.9882644414901733, + "val/ratio_var": 6.539086461998522e-05 + }, + { + "episode": 15376, + "epoch": 0.7765656565656566, + "eps": 0, + "loss/policy_avg": -0.0476013720035553, + "loss/value_avg": 0.682388961315155, + "lr": 1.3166227550391542e-06, + "objective/entropy": -97.00421142578125, + "objective/kl": 42.212120056152344, + "objective/non_score_reward": -2.1106061935424805, + "objective/rlhf_reward": 13.99876880645752, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000946523156017065, + "policy/clipfrac_avg": 0.0059087686240673065, + "policy/entropy_avg": 0.19147014617919922, + "step": 961, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 224, + "val/ratio": 0.9874411225318909, + "val/ratio_var": 5.3472565923584625e-05 + }, + { + "episode": 15392, + "epoch": 0.7773737373737374, + "eps": 0, + "loss/policy_avg": -0.26426494121551514, + "loss/value_avg": 0.47233304381370544, + "lr": 1.307603209449378e-06, + "objective/entropy": -60.28771209716797, + "objective/kl": 39.21135330200195, + "objective/non_score_reward": -1.9605677127838135, + "objective/rlhf_reward": 15.351932525634766, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009600625489838421, + "policy/clipfrac_avg": 0.0035959878005087376, + "policy/entropy_avg": 0.18239498138427734, + "step": 962, + "val/clipfrac_avg": 0.012040323577821255, + "val/num_eos_tokens": 296, + "val/ratio": 0.9875339269638062, + "val/ratio_var": 6.485576159320772e-05 + }, + { + "episode": 15408, + "epoch": 0.7781818181818182, + "eps": 0, + "loss/policy_avg": 0.04519583284854889, + "loss/value_avg": 0.7595553398132324, + "lr": 1.298610014623423e-06, + "objective/entropy": -146.63661193847656, + "objective/kl": 38.37220001220703, + "objective/non_score_reward": -1.9186100959777832, + "objective/rlhf_reward": 13.354827880859375, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007201317930594087, + "policy/clipfrac_avg": 0.0022899312898516655, + "policy/entropy_avg": 0.1562885046005249, + "step": 963, + "val/clipfrac_avg": 8.292270649690181e-05, + "val/num_eos_tokens": 521, + "val/ratio": 0.9906989336013794, + "val/ratio_var": 6.185618985909969e-05 + }, + { + "episode": 15424, + "epoch": 0.778989898989899, + "eps": 0, + "loss/policy_avg": -0.14623457193374634, + "loss/value_avg": 0.35101252794265747, + "lr": 1.2896432347411553e-06, + "objective/entropy": -66.08819580078125, + "objective/kl": 36.48185348510742, + "objective/non_score_reward": -1.8240927457809448, + "objective/rlhf_reward": 14.261844635009766, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009832159848883748, + "policy/clipfrac_avg": 0.006735666189342737, + "policy/entropy_avg": 0.19070911407470703, + "step": 964, + "val/clipfrac_avg": 0.0030241934582591057, + "val/num_eos_tokens": 421, + "val/ratio": 0.9872285723686218, + "val/ratio_var": 7.909470878075808e-05 + }, + { + "episode": 15440, + "epoch": 0.7797979797979798, + "eps": 0, + "loss/policy_avg": -0.1669466346502304, + "loss/value_avg": 0.6373587846755981, + "lr": 1.2807029337939403e-06, + "objective/entropy": -48.76942825317383, + "objective/kl": 36.51701354980469, + "objective/non_score_reward": -1.8258507251739502, + "objective/rlhf_reward": 13.861648559570312, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.000836203049402684, + "policy/clipfrac_avg": 0.006686762906610966, + "policy/entropy_avg": 0.17627906799316406, + "step": 965, + "val/clipfrac_avg": 0.009943181648850441, + "val/num_eos_tokens": 11, + "val/ratio": 0.9892607927322388, + "val/ratio_var": 5.405569390859455e-05 + }, + { + "episode": 15456, + "epoch": 0.7806060606060606, + "eps": 0, + "loss/policy_avg": -0.005087921395897865, + "loss/value_avg": 0.2712212800979614, + "lr": 1.2717891755841722e-06, + "objective/entropy": -59.34945297241211, + "objective/kl": 42.5623893737793, + "objective/non_score_reward": -2.128119468688965, + "objective/rlhf_reward": 13.918755531311035, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008710323600098491, + "policy/clipfrac_avg": 0.010207775980234146, + "policy/entropy_avg": 0.1739978790283203, + "step": 966, + "val/clipfrac_avg": 0.010802468284964561, + "val/num_eos_tokens": 246, + "val/ratio": 0.9884668588638306, + "val/ratio_var": 4.567186624626629e-05 + }, + { + "episode": 15472, + "epoch": 0.7814141414141414, + "eps": 0, + "loss/policy_avg": -0.5036620497703552, + "loss/value_avg": 0.39513152837753296, + "lr": 1.2629020237248241e-06, + "objective/entropy": -136.59652709960938, + "objective/kl": 29.273542404174805, + "objective/non_score_reward": -1.463677167892456, + "objective/rlhf_reward": 14.083197593688965, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0007826394285075366, + "policy/clipfrac_avg": 0.006299240980297327, + "policy/entropy_avg": 0.15417519211769104, + "step": 967, + "val/clipfrac_avg": 0.03271849453449249, + "val/num_eos_tokens": 249, + "val/ratio": 0.9902065992355347, + "val/ratio_var": 8.734448783798143e-05 + }, + { + "episode": 15488, + "epoch": 0.7822222222222223, + "eps": 0, + "loss/policy_avg": -0.04834941402077675, + "loss/value_avg": 0.49886178970336914, + "lr": 1.2540415416389917e-06, + "objective/entropy": -29.31902503967285, + "objective/kl": 38.655548095703125, + "objective/non_score_reward": -1.9327776432037354, + "objective/rlhf_reward": 13.864096641540527, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008980755810625851, + "policy/clipfrac_avg": 0.003656789194792509, + "policy/entropy_avg": 0.1847667694091797, + "step": 968, + "val/clipfrac_avg": 5.3879310144111514e-05, + "val/num_eos_tokens": 10, + "val/ratio": 0.9879658818244934, + "val/ratio_var": 2.3956074073794298e-05 + }, + { + "episode": 15504, + "epoch": 0.7830303030303031, + "eps": 0, + "loss/policy_avg": -0.2217503786087036, + "loss/value_avg": 0.4100498557090759, + "lr": 1.2452077925594435e-06, + "objective/entropy": -206.95504760742188, + "objective/kl": 41.82144546508789, + "objective/non_score_reward": -2.0910723209381104, + "objective/rlhf_reward": 13.076896667480469, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008509804611094296, + "policy/clipfrac_avg": 0.0047798133455216885, + "policy/entropy_avg": 0.15673446655273438, + "step": 969, + "val/clipfrac_avg": 0.00021062050655018538, + "val/num_eos_tokens": 481, + "val/ratio": 0.9887630939483643, + "val/ratio_var": 6.604092777706683e-05 + }, + { + "episode": 15520, + "epoch": 0.7838383838383839, + "eps": 0, + "loss/policy_avg": -0.02120068296790123, + "loss/value_avg": 0.6901993751525879, + "lr": 1.23640083952817e-06, + "objective/entropy": -40.900238037109375, + "objective/kl": 38.07307052612305, + "objective/non_score_reward": -1.9036535024642944, + "objective/rlhf_reward": 14.377596855163574, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010727059561759233, + "policy/clipfrac_avg": 0.004337977152317762, + "policy/entropy_avg": 0.1984100341796875, + "step": 970, + "val/clipfrac_avg": 0.02858383022248745, + "val/num_eos_tokens": 535, + "val/ratio": 0.9863028526306152, + "val/ratio_var": 8.229557715822011e-05 + }, + { + "episode": 15536, + "epoch": 0.7846464646464646, + "eps": 0, + "loss/policy_avg": -0.09917282313108444, + "loss/value_avg": 0.3991038203239441, + "lr": 1.2276207453959283e-06, + "objective/entropy": -77.6534652709961, + "objective/kl": 39.35569381713867, + "objective/non_score_reward": -1.9677847623825073, + "objective/rlhf_reward": 13.907215118408203, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008703683270141482, + "policy/clipfrac_avg": 0.0043828776106238365, + "policy/entropy_avg": 0.17210960388183594, + "step": 971, + "val/clipfrac_avg": 0.00012400794366840273, + "val/num_eos_tokens": 175, + "val/ratio": 0.988074541091919, + "val/ratio_var": 5.318454350344837e-05 + }, + { + "episode": 15552, + "epoch": 0.7854545454545454, + "eps": 0, + "loss/policy_avg": 0.0814996063709259, + "loss/value_avg": 0.6175538897514343, + "lr": 1.2188675728217986e-06, + "objective/entropy": -16.92827606201172, + "objective/kl": 41.14687728881836, + "objective/non_score_reward": -2.0573439598083496, + "objective/rlhf_reward": 15.442655563354492, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010900306515395641, + "policy/clipfrac_avg": 0.00866631604731083, + "policy/entropy_avg": 0.2063446044921875, + "step": 972, + "val/clipfrac_avg": 0.009025804698467255, + "val/num_eos_tokens": 174, + "val/ratio": 0.9854716062545776, + "val/ratio_var": 2.7480951757752337e-05 + }, + { + "episode": 15568, + "epoch": 0.7862626262626262, + "eps": 0, + "loss/policy_avg": 0.3593114912509918, + "loss/value_avg": 0.7372395396232605, + "lr": 1.2101413842727345e-06, + "objective/entropy": -63.132266998291016, + "objective/kl": 45.04676818847656, + "objective/non_score_reward": -2.252338409423828, + "objective/rlhf_reward": 14.450786590576172, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009616734459996223, + "policy/clipfrac_avg": 0.0046102688647806644, + "policy/entropy_avg": 0.1856832504272461, + "step": 973, + "val/clipfrac_avg": 0.016889449208974838, + "val/num_eos_tokens": 419, + "val/ratio": 0.9877485036849976, + "val/ratio_var": 6.443963502533734e-05 + }, + { + "episode": 15584, + "epoch": 0.787070707070707, + "eps": 0, + "loss/policy_avg": 0.14888805150985718, + "loss/value_avg": 0.7544107437133789, + "lr": 1.201442242023122e-06, + "objective/entropy": -98.38099670410156, + "objective/kl": 41.9028434753418, + "objective/non_score_reward": -2.095142126083374, + "objective/rlhf_reward": 12.186107635498047, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009328118758276105, + "policy/clipfrac_avg": 0.004906289279460907, + "policy/entropy_avg": 0.17488479614257812, + "step": 974, + "val/clipfrac_avg": 0.08914318680763245, + "val/num_eos_tokens": 193, + "val/ratio": 0.9888035655021667, + "val/ratio_var": 6.763465353287756e-05 + }, + { + "episode": 15600, + "epoch": 0.7878787878787878, + "eps": 0, + "loss/policy_avg": -0.05416979640722275, + "loss/value_avg": 0.5626612901687622, + "lr": 1.1927702081543279e-06, + "objective/entropy": -65.54731750488281, + "objective/kl": 33.80537796020508, + "objective/non_score_reward": -1.6902689933776855, + "objective/rlhf_reward": 13.434730529785156, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010051055578514934, + "policy/clipfrac_avg": 0.005396673921495676, + "policy/entropy_avg": 0.19644975662231445, + "step": 975, + "val/clipfrac_avg": 0.02293882891535759, + "val/num_eos_tokens": 539, + "val/ratio": 0.9861206412315369, + "val/ratio_var": 7.48153543099761e-05 + }, + { + "episode": 15616, + "epoch": 0.7886868686868687, + "eps": 0, + "loss/policy_avg": 0.05607307702302933, + "loss/value_avg": 0.48446062207221985, + "lr": 1.1841253445542605e-06, + "objective/entropy": -29.001220703125, + "objective/kl": 36.37082290649414, + "objective/non_score_reward": -1.8185412883758545, + "objective/rlhf_reward": 12.142396926879883, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0011397147318348289, + "policy/clipfrac_avg": 0.0031259057577699423, + "policy/entropy_avg": 0.22170257568359375, + "step": 976, + "val/clipfrac_avg": 0.005424283444881439, + "val/num_eos_tokens": 387, + "val/ratio": 0.9855947494506836, + "val/ratio_var": 6.33225790807046e-05 + }, + { + "episode": 15632, + "epoch": 0.7894949494949495, + "eps": 0, + "loss/policy_avg": -0.15400168299674988, + "loss/value_avg": 0.8165977001190186, + "lr": 1.1755077129169284e-06, + "objective/entropy": -60.56960678100586, + "objective/kl": 37.47486114501953, + "objective/non_score_reward": -1.8737430572509766, + "objective/rlhf_reward": 11.046178817749023, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0009815995581448078, + "policy/clipfrac_avg": 0.005213981028646231, + "policy/entropy_avg": 0.21601486206054688, + "step": 977, + "val/clipfrac_avg": 0.022167012095451355, + "val/num_eos_tokens": 93, + "val/ratio": 0.9861747026443481, + "val/ratio_var": 7.86496966611594e-05 + }, + { + "episode": 15648, + "epoch": 0.7903030303030303, + "eps": 0, + "loss/policy_avg": -0.1725654900074005, + "loss/value_avg": 0.487238347530365, + "lr": 1.166917374742e-06, + "objective/entropy": -56.807586669921875, + "objective/kl": 37.225948333740234, + "objective/non_score_reward": -1.861297607421875, + "objective/rlhf_reward": 14.388702392578125, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102808338124305, + "policy/clipfrac_avg": 0.0037505230866372585, + "policy/entropy_avg": 0.20328998565673828, + "step": 978, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 128, + "val/ratio": 0.9871898293495178, + "val/ratio_var": 9.372083150083199e-05 + }, + { + "episode": 15664, + "epoch": 0.7911111111111111, + "eps": 0, + "loss/policy_avg": -0.2764543294906616, + "loss/value_avg": 0.33185577392578125, + "lr": 1.158354391334362e-06, + "objective/entropy": -78.30638122558594, + "objective/kl": 34.59331512451172, + "objective/non_score_reward": -1.729665756225586, + "objective/rlhf_reward": 15.489084243774414, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008904373971745372, + "policy/clipfrac_avg": 0.00305290799587965, + "policy/entropy_avg": 0.17419052124023438, + "step": 979, + "val/clipfrac_avg": 0.09470640122890472, + "val/num_eos_tokens": 367, + "val/ratio": 0.9881394505500793, + "val/ratio_var": 6.588637916138396e-05 + }, + { + "episode": 15680, + "epoch": 0.7919191919191919, + "eps": 0, + "loss/policy_avg": 0.049546245485544205, + "loss/value_avg": 0.5992924571037292, + "lr": 1.1498188238036862e-06, + "objective/entropy": -146.6639862060547, + "objective/kl": 40.94211959838867, + "objective/non_score_reward": -2.0471062660217285, + "objective/rlhf_reward": 12.40992546081543, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008619745494797826, + "policy/clipfrac_avg": 0.007621705532073975, + "policy/entropy_avg": 0.15411758422851562, + "step": 980, + "val/clipfrac_avg": 0.09697519987821579, + "val/num_eos_tokens": 405, + "val/ratio": 0.9889684915542603, + "val/ratio_var": 5.157575287739746e-05 + }, + { + "episode": 15696, + "epoch": 0.7927272727272727, + "eps": 0, + "loss/policy_avg": 0.7101428508758545, + "loss/value_avg": 1.0345678329467773, + "lr": 1.141310733063991e-06, + "objective/entropy": -111.55574798583984, + "objective/kl": 35.79730224609375, + "objective/non_score_reward": -1.7898651361465454, + "objective/rlhf_reward": 13.764822006225586, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009224195964634418, + "policy/clipfrac_avg": 0.00732937827706337, + "policy/entropy_avg": 0.1737055778503418, + "step": 981, + "val/clipfrac_avg": 0.0015625000232830644, + "val/num_eos_tokens": 230, + "val/ratio": 0.9885663390159607, + "val/ratio_var": 7.395897409878671e-05 + }, + { + "episode": 15712, + "epoch": 0.7935353535353535, + "eps": 0, + "loss/policy_avg": -0.1941525638103485, + "loss/value_avg": 0.4167664647102356, + "lr": 1.132830179833207e-06, + "objective/entropy": -136.7578125, + "objective/kl": 41.060279846191406, + "objective/non_score_reward": -2.053014039993286, + "objective/rlhf_reward": 14.564172744750977, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008562670554965734, + "policy/clipfrac_avg": 0.005737176164984703, + "policy/entropy_avg": 0.1706523895263672, + "step": 982, + "val/clipfrac_avg": 0.006640523672103882, + "val/num_eos_tokens": 935, + "val/ratio": 0.9893268346786499, + "val/ratio_var": 6.982601917115971e-05 + }, + { + "episode": 15728, + "epoch": 0.7943434343434344, + "eps": 0, + "loss/policy_avg": -0.1428111046552658, + "loss/value_avg": 0.2174670249223709, + "lr": 1.1243772246327416e-06, + "objective/entropy": -109.03129577636719, + "objective/kl": 38.72102355957031, + "objective/non_score_reward": -1.9360511302947998, + "objective/rlhf_reward": 14.470198631286621, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000771217979490757, + "policy/clipfrac_avg": 0.002998888725414872, + "policy/entropy_avg": 0.16396713256835938, + "step": 983, + "val/clipfrac_avg": 0.020510124042630196, + "val/num_eos_tokens": 25, + "val/ratio": 0.989021897315979, + "val/ratio_var": 4.0093269490171224e-05 + }, + { + "episode": 15744, + "epoch": 0.7951515151515152, + "eps": 0, + "loss/policy_avg": 0.010445185005664825, + "loss/value_avg": 0.37888824939727783, + "lr": 1.1159519277870507e-06, + "objective/entropy": -6.635685920715332, + "objective/kl": 38.89892578125, + "objective/non_score_reward": -1.9449464082717896, + "objective/rlhf_reward": 14.7113037109375, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009513833792880177, + "policy/clipfrac_avg": 0.003157425671815872, + "policy/entropy_avg": 0.1973400115966797, + "step": 984, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 21, + "val/ratio": 0.9875931739807129, + "val/ratio_var": 5.873597910976969e-05 + }, + { + "episode": 15760, + "epoch": 0.795959595959596, + "eps": 0, + "loss/policy_avg": -0.28500208258628845, + "loss/value_avg": 0.4213254451751709, + "lr": 1.1075543494232049e-06, + "objective/entropy": -149.3587646484375, + "objective/kl": 31.575908660888672, + "objective/non_score_reward": -1.5787954330444336, + "objective/rlhf_reward": 15.75714111328125, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007162499241530895, + "policy/clipfrac_avg": 0.0078894617035985, + "policy/entropy_avg": 0.14484691619873047, + "step": 985, + "val/clipfrac_avg": 0.023666253313422203, + "val/num_eos_tokens": 155, + "val/ratio": 0.9908950328826904, + "val/ratio_var": 5.517086901818402e-05 + }, + { + "episode": 15776, + "epoch": 0.7967676767676768, + "eps": 0, + "loss/policy_avg": -0.016947008669376373, + "loss/value_avg": 0.3959222733974457, + "lr": 1.0991845494704673e-06, + "objective/entropy": -90.30593872070312, + "objective/kl": 44.143898010253906, + "objective/non_score_reward": -2.2071948051452637, + "objective/rlhf_reward": 12.292805671691895, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000890448980499059, + "policy/clipfrac_avg": 0.005722125992178917, + "policy/entropy_avg": 0.1587371826171875, + "step": 986, + "val/clipfrac_avg": 0.020597470924258232, + "val/num_eos_tokens": 327, + "val/ratio": 0.9887195825576782, + "val/ratio_var": 5.094557855045423e-05 + }, + { + "episode": 15792, + "epoch": 0.7975757575757576, + "eps": 0, + "loss/policy_avg": 0.022888287901878357, + "loss/value_avg": 0.36602234840393066, + "lr": 1.0908425876598512e-06, + "objective/entropy": -140.4935760498047, + "objective/kl": 42.647743225097656, + "objective/non_score_reward": -2.132387161254883, + "objective/rlhf_reward": 13.516050338745117, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008821362862363458, + "policy/clipfrac_avg": 0.005631787236779928, + "policy/entropy_avg": 0.1700587272644043, + "step": 987, + "val/clipfrac_avg": 0.012090040370821953, + "val/num_eos_tokens": 546, + "val/ratio": 0.9886399507522583, + "val/ratio_var": 5.682192204403691e-05 + }, + { + "episode": 15808, + "epoch": 0.7983838383838384, + "eps": 0, + "loss/policy_avg": 0.19011469185352325, + "loss/value_avg": 0.5114719271659851, + "lr": 1.082528523523713e-06, + "objective/entropy": -53.488460540771484, + "objective/kl": 36.984073638916016, + "objective/non_score_reward": -1.8492038249969482, + "objective/rlhf_reward": 14.049233436584473, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009789207251742482, + "policy/clipfrac_avg": 0.005884348414838314, + "policy/entropy_avg": 0.20690059661865234, + "step": 988, + "val/clipfrac_avg": 0.013353430666029453, + "val/num_eos_tokens": 353, + "val/ratio": 0.9872037768363953, + "val/ratio_var": 6.569022662006319e-05 + }, + { + "episode": 15824, + "epoch": 0.7991919191919192, + "eps": 0, + "loss/policy_avg": 0.1753331869840622, + "loss/value_avg": 0.3773725628852844, + "lr": 1.0742424163953096e-06, + "objective/entropy": -56.68682861328125, + "objective/kl": 37.34782409667969, + "objective/non_score_reward": -1.8673913478851318, + "objective/rlhf_reward": 13.050577163696289, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.000838876876514405, + "policy/clipfrac_avg": 0.0064134905114769936, + "policy/entropy_avg": 0.16259098052978516, + "step": 989, + "val/clipfrac_avg": 0.055415935814380646, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885395169258118, + "val/ratio_var": 8.253273699665442e-05 + }, + { + "episode": 15840, + "epoch": 0.8, + "eps": 0, + "loss/policy_avg": -0.013544011861085892, + "loss/value_avg": 0.5490726828575134, + "lr": 1.0659843254083919e-06, + "objective/entropy": -84.58714294433594, + "objective/kl": 34.28784942626953, + "objective/non_score_reward": -1.7143926620483398, + "objective/rlhf_reward": 13.20748233795166, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009669119026511908, + "policy/clipfrac_avg": 0.00355747202411294, + "policy/entropy_avg": 0.18613052368164062, + "step": 990, + "val/clipfrac_avg": 0.015838205814361572, + "val/num_eos_tokens": 115, + "val/ratio": 0.9877301454544067, + "val/ratio_var": 5.967232209513895e-05 + }, + { + "episode": 15856, + "epoch": 0.8008080808080809, + "eps": 0, + "loss/policy_avg": 0.17707303166389465, + "loss/value_avg": 0.24937717616558075, + "lr": 1.0577543094967613e-06, + "objective/entropy": -134.7545928955078, + "objective/kl": 35.57737731933594, + "objective/non_score_reward": -1.7788686752319336, + "objective/rlhf_reward": 15.010193824768066, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008520459523424506, + "policy/clipfrac_avg": 0.007036273833364248, + "policy/entropy_avg": 0.14983177185058594, + "step": 991, + "val/clipfrac_avg": 0.0005040322430431843, + "val/num_eos_tokens": 450, + "val/ratio": 0.9898406863212585, + "val/ratio_var": 4.8867601435631514e-05 + }, + { + "episode": 15872, + "epoch": 0.8016161616161617, + "eps": 0, + "loss/policy_avg": -0.1231779083609581, + "loss/value_avg": 0.27385565638542175, + "lr": 1.0495524273938735e-06, + "objective/entropy": -99.90908813476562, + "objective/kl": 37.30508804321289, + "objective/non_score_reward": -1.865254521369934, + "objective/rlhf_reward": 12.759745597839355, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008709601243026555, + "policy/clipfrac_avg": 0.004131123423576355, + "policy/entropy_avg": 0.17008209228515625, + "step": 992, + "val/clipfrac_avg": 0.01071952749043703, + "val/num_eos_tokens": 376, + "val/ratio": 0.9894505739212036, + "val/ratio_var": 6.905445479787886e-05 + }, + { + "episode": 15888, + "epoch": 0.8024242424242424, + "eps": 0, + "loss/policy_avg": -0.11878392100334167, + "loss/value_avg": 0.6012347340583801, + "lr": 1.041378737632402e-06, + "objective/entropy": -89.06427764892578, + "objective/kl": 39.19951629638672, + "objective/non_score_reward": -1.9599759578704834, + "objective/rlhf_reward": 14.532211303710938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009216031758114696, + "policy/clipfrac_avg": 0.00672511849552393, + "policy/entropy_avg": 0.1765270233154297, + "step": 993, + "val/clipfrac_avg": 0.0031250000465661287, + "val/num_eos_tokens": 74, + "val/ratio": 0.9882692098617554, + "val/ratio_var": 3.506496432237327e-05 + }, + { + "episode": 15904, + "epoch": 0.8032323232323232, + "eps": 0, + "loss/policy_avg": 0.11932457238435745, + "loss/value_avg": 0.28720468282699585, + "lr": 1.0332332985438248e-06, + "objective/entropy": -48.50525665283203, + "objective/kl": 37.94336700439453, + "objective/non_score_reward": -1.8971682786941528, + "objective/rlhf_reward": 15.641894340515137, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010036650346592069, + "policy/clipfrac_avg": 0.0061247204430401325, + "policy/entropy_avg": 0.18225669860839844, + "step": 994, + "val/clipfrac_avg": 0.01019021775573492, + "val/num_eos_tokens": 8, + "val/ratio": 0.9874961972236633, + "val/ratio_var": 5.3736544941784814e-05 + }, + { + "episode": 15920, + "epoch": 0.804040404040404, + "eps": 0, + "loss/policy_avg": 0.14462614059448242, + "loss/value_avg": 0.3247252106666565, + "lr": 1.0251161682580125e-06, + "objective/entropy": -108.57736206054688, + "objective/kl": 41.1889762878418, + "objective/non_score_reward": -2.059448719024658, + "objective/rlhf_reward": 14.182738304138184, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001018329057842493, + "policy/clipfrac_avg": 0.0034168935380876064, + "policy/entropy_avg": 0.20872879028320312, + "step": 995, + "val/clipfrac_avg": 0.00548538425937295, + "val/num_eos_tokens": 8, + "val/ratio": 0.9865425825119019, + "val/ratio_var": 8.348035044036806e-05 + }, + { + "episode": 15936, + "epoch": 0.8048484848484848, + "eps": 0, + "loss/policy_avg": 0.2817779779434204, + "loss/value_avg": 0.49794071912765503, + "lr": 1.0170274047028068e-06, + "objective/entropy": -43.08762741088867, + "objective/kl": 45.49590301513672, + "objective/non_score_reward": -2.2747950553894043, + "objective/rlhf_reward": 13.912704467773438, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010373364202678204, + "policy/clipfrac_avg": 0.006608482450246811, + "policy/entropy_avg": 0.2166609764099121, + "step": 996, + "val/clipfrac_avg": 0.03890983760356903, + "val/num_eos_tokens": 11, + "val/ratio": 0.986150860786438, + "val/ratio_var": 8.553406951250508e-05 + }, + { + "episode": 15952, + "epoch": 0.8056565656565656, + "eps": 0, + "loss/policy_avg": -0.22047999501228333, + "loss/value_avg": 0.5745018720626831, + "lr": 1.0089670656036137e-06, + "objective/entropy": -114.1541976928711, + "objective/kl": 35.899574279785156, + "objective/non_score_reward": -1.7949787378311157, + "objective/rlhf_reward": 14.869083404541016, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007367507787421346, + "policy/clipfrac_avg": 0.0037586798425763845, + "policy/entropy_avg": 0.15613555908203125, + "step": 997, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 560, + "val/ratio": 0.9902479648590088, + "val/ratio_var": 5.6676559324841946e-05 + }, + { + "episode": 15968, + "epoch": 0.8064646464646464, + "eps": 0, + "loss/policy_avg": -0.04391951486468315, + "loss/value_avg": 0.6516869068145752, + "lr": 1.0009352084829871e-06, + "objective/entropy": -112.42098999023438, + "objective/kl": 39.436988830566406, + "objective/non_score_reward": -1.9718495607376099, + "objective/rlhf_reward": 14.48127555847168, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009028672939166427, + "policy/clipfrac_avg": 0.0059207589365541935, + "policy/entropy_avg": 0.18039512634277344, + "step": 998, + "val/clipfrac_avg": 0.0127427838742733, + "val/num_eos_tokens": 515, + "val/ratio": 0.9878066778182983, + "val/ratio_var": 8.427277498412877e-05 + }, + { + "episode": 15984, + "epoch": 0.8072727272727273, + "eps": 0, + "loss/policy_avg": -0.5726251602172852, + "loss/value_avg": 0.6322129964828491, + "lr": 9.929318906602176e-07, + "objective/entropy": -115.74237060546875, + "objective/kl": 35.111915588378906, + "objective/non_score_reward": -1.7555959224700928, + "objective/rlhf_reward": 14.103778839111328, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006586643867194653, + "policy/clipfrac_avg": 0.006481423042714596, + "policy/entropy_avg": 0.1394972801208496, + "step": 999, + "val/clipfrac_avg": 0.019941631704568863, + "val/num_eos_tokens": 81, + "val/ratio": 0.9906902313232422, + "val/ratio_var": 5.2608244004659355e-05 + }, + { + "episode": 16000, + "epoch": 0.8080808080808081, + "eps": 0, + "loss/policy_avg": -0.1571139395236969, + "loss/value_avg": 0.8596901297569275, + "lr": 9.849571692509257e-07, + "objective/entropy": -35.66822052001953, + "objective/kl": 36.119667053222656, + "objective/non_score_reward": -1.805983304977417, + "objective/rlhf_reward": 12.842453956604004, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008497819071635604, + "policy/clipfrac_avg": 0.0041199941188097, + "policy/entropy_avg": 0.16556644439697266, + "step": 1000, + "val/clipfrac_avg": 0.022714650258421898, + "val/num_eos_tokens": 82, + "val/ratio": 0.9879409074783325, + "val/ratio_var": 6.342474807752296e-05 + }, + { + "episode": 16016, + "epoch": 0.8088888888888889, + "eps": 0, + "loss/policy_avg": -0.10610047727823257, + "loss/value_avg": 0.5593327283859253, + "lr": 9.770111011666582e-07, + "objective/entropy": -112.11347961425781, + "objective/kl": 33.91107177734375, + "objective/non_score_reward": -1.6955535411834717, + "objective/rlhf_reward": 13.304447174072266, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008447067812085152, + "policy/clipfrac_avg": 0.004359187558293343, + "policy/entropy_avg": 0.18085873126983643, + "step": 1001, + "val/clipfrac_avg": 0.0036525975447148085, + "val/num_eos_tokens": 37, + "val/ratio": 0.9888993501663208, + "val/ratio_var": 0.00012090770906070247 + }, + { + "episode": 16032, + "epoch": 0.8096969696969697, + "eps": 0, + "loss/policy_avg": -0.4011276960372925, + "loss/value_avg": 0.8269133567810059, + "lr": 9.690937431144725e-07, + "objective/entropy": -143.82159423828125, + "objective/kl": 32.61180877685547, + "objective/non_score_reward": -1.6305904388427734, + "objective/rlhf_reward": 13.322534561157227, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0007577177020721138, + "policy/clipfrac_avg": 0.005468646064400673, + "policy/entropy_avg": 0.14389371871948242, + "step": 1002, + "val/clipfrac_avg": 0.016304347664117813, + "val/num_eos_tokens": 153, + "val/ratio": 0.990355908870697, + "val/ratio_var": 4.9998925533145666e-05 + }, + { + "episode": 16048, + "epoch": 0.8105050505050505, + "eps": 0, + "loss/policy_avg": -0.18046468496322632, + "loss/value_avg": 0.4406319260597229, + "lr": 9.612051515965388e-07, + "objective/entropy": -41.019737243652344, + "objective/kl": 43.84602355957031, + "objective/non_score_reward": -2.1923012733459473, + "objective/rlhf_reward": 14.870198249816895, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009248010464943945, + "policy/clipfrac_avg": 0.004084739368408918, + "policy/entropy_avg": 0.1930408477783203, + "step": 1003, + "val/clipfrac_avg": 0.01253926008939743, + "val/num_eos_tokens": 425, + "val/ratio": 0.9877877235412598, + "val/ratio_var": 4.6758981625316665e-05 + }, + { + "episode": 16064, + "epoch": 0.8113131313131313, + "eps": 0, + "loss/policy_avg": -0.03701537847518921, + "loss/value_avg": 0.4312940835952759, + "lr": 9.533453829097355e-07, + "objective/entropy": -115.09342956542969, + "objective/kl": 43.825538635253906, + "objective/non_score_reward": -2.191277027130127, + "objective/rlhf_reward": 14.941534996032715, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008623325265944004, + "policy/clipfrac_avg": 0.004582014866173267, + "policy/entropy_avg": 0.17541944980621338, + "step": 1004, + "val/clipfrac_avg": 0.02018960565328598, + "val/num_eos_tokens": 259, + "val/ratio": 0.988018274307251, + "val/ratio_var": 5.916683949180879e-05 + }, + { + "episode": 16080, + "epoch": 0.8121212121212121, + "eps": 0, + "loss/policy_avg": 0.30280348658561707, + "loss/value_avg": 0.7898949384689331, + "lr": 9.455144931452459e-07, + "objective/entropy": -62.099971771240234, + "objective/kl": 30.437484741210938, + "objective/non_score_reward": -1.5218743085861206, + "objective/rlhf_reward": 13.712501525878906, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007971136947162449, + "policy/clipfrac_avg": 0.008183586411178112, + "policy/entropy_avg": 0.1460132598876953, + "step": 1005, + "val/clipfrac_avg": 0.003989361692219973, + "val/num_eos_tokens": 215, + "val/ratio": 0.9896875023841858, + "val/ratio_var": 4.330440060584806e-05 + }, + { + "episode": 16096, + "epoch": 0.812929292929293, + "eps": 0, + "loss/policy_avg": -0.23599471151828766, + "loss/value_avg": 1.1437101364135742, + "lr": 9.377125381881607e-07, + "objective/entropy": -46.674827575683594, + "objective/kl": 33.718963623046875, + "objective/non_score_reward": -1.6859480142593384, + "objective/rlhf_reward": 14.86874008178711, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.00085799815133214, + "policy/clipfrac_avg": 0.003210378112271428, + "policy/entropy_avg": 0.19154703617095947, + "step": 1006, + "val/clipfrac_avg": 0.010907451622188091, + "val/num_eos_tokens": 259, + "val/ratio": 0.9871425628662109, + "val/ratio_var": 6.926390778971836e-05 + }, + { + "episode": 16112, + "epoch": 0.8137373737373738, + "eps": 0, + "loss/policy_avg": -0.02023145742714405, + "loss/value_avg": 1.2994359731674194, + "lr": 9.299395737170758e-07, + "objective/entropy": -87.60648345947266, + "objective/kl": 35.53752899169922, + "objective/non_score_reward": -1.7768765687942505, + "objective/rlhf_reward": 15.644998550415039, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007149496814236045, + "policy/clipfrac_avg": 0.002271715085953474, + "policy/entropy_avg": 0.1492328643798828, + "step": 1007, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 12, + "val/ratio": 0.9887728691101074, + "val/ratio_var": 5.388016870710999e-05 + }, + { + "episode": 16128, + "epoch": 0.8145454545454546, + "eps": 0, + "loss/policy_avg": 0.022628188133239746, + "loss/value_avg": 1.1084355115890503, + "lr": 9.221956552036992e-07, + "objective/entropy": -63.064239501953125, + "objective/kl": 36.12407302856445, + "objective/non_score_reward": -1.8062036037445068, + "objective/rlhf_reward": 15.146921157836914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008970820344984531, + "policy/clipfrac_avg": 0.004715266637504101, + "policy/entropy_avg": 0.1754002571105957, + "step": 1008, + "val/clipfrac_avg": 0.008854166604578495, + "val/num_eos_tokens": 540, + "val/ratio": 0.9880603551864624, + "val/ratio_var": 8.214469562517479e-05 + }, + { + "episode": 16144, + "epoch": 0.8153535353535354, + "eps": 0, + "loss/policy_avg": -0.06129603087902069, + "loss/value_avg": 0.558459997177124, + "lr": 9.144808379124509e-07, + "objective/entropy": -104.08320617675781, + "objective/kl": 38.30896759033203, + "objective/non_score_reward": -1.915448546409607, + "objective/rlhf_reward": 13.201739311218262, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008289464749395847, + "policy/clipfrac_avg": 0.0036508054472506046, + "policy/entropy_avg": 0.16502058506011963, + "step": 1009, + "val/clipfrac_avg": 0.013535099104046822, + "val/num_eos_tokens": 360, + "val/ratio": 0.9892135858535767, + "val/ratio_var": 6.506205681944266e-05 + }, + { + "episode": 16160, + "epoch": 0.8161616161616162, + "eps": 0, + "loss/policy_avg": -0.06057693436741829, + "loss/value_avg": 0.288950115442276, + "lr": 9.067951769000699e-07, + "objective/entropy": -71.51614379882812, + "objective/kl": 42.83964538574219, + "objective/non_score_reward": -2.141982078552246, + "objective/rlhf_reward": 14.701767921447754, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009444506140425801, + "policy/clipfrac_avg": 0.002944172825664282, + "policy/entropy_avg": 0.1825122833251953, + "step": 1010, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 613, + "val/ratio": 0.9874241352081299, + "val/ratio_var": 3.9652099076192826e-05 + }, + { + "episode": 16176, + "epoch": 0.816969696969697, + "eps": 0, + "loss/policy_avg": -0.0031431764364242554, + "loss/value_avg": 0.5874146223068237, + "lr": 8.991387270152202e-07, + "objective/entropy": -63.48155212402344, + "objective/kl": 43.46244812011719, + "objective/non_score_reward": -2.1731224060058594, + "objective/rlhf_reward": 14.65500259399414, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009073936380445957, + "policy/clipfrac_avg": 0.005083385389298201, + "policy/entropy_avg": 0.1724100112915039, + "step": 1011, + "val/clipfrac_avg": 0.0003350692568346858, + "val/num_eos_tokens": 564, + "val/ratio": 0.9878172874450684, + "val/ratio_var": 5.2520466852001846e-05 + }, + { + "episode": 16192, + "epoch": 0.8177777777777778, + "eps": 0, + "loss/policy_avg": 0.020213183015584946, + "loss/value_avg": 0.3557816445827484, + "lr": 8.915115428981064e-07, + "objective/entropy": -24.23276138305664, + "objective/kl": 30.50366973876953, + "objective/non_score_reward": -1.5251834392547607, + "objective/rlhf_reward": 14.45919132232666, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009571783011779189, + "policy/clipfrac_avg": 0.004434539936482906, + "policy/entropy_avg": 0.18828392028808594, + "step": 1012, + "val/clipfrac_avg": 0.015398487448692322, + "val/num_eos_tokens": 104, + "val/ratio": 0.9878268241882324, + "val/ratio_var": 6.845602911198512e-05 + }, + { + "episode": 16208, + "epoch": 0.8185858585858586, + "eps": 0, + "loss/policy_avg": -0.06331466138362885, + "loss/value_avg": 0.6016228795051575, + "lr": 8.839136789800729e-07, + "objective/entropy": -72.25382995605469, + "objective/kl": 35.868072509765625, + "objective/non_score_reward": -1.7934037446975708, + "objective/rlhf_reward": 14.183158874511719, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008664189372211695, + "policy/clipfrac_avg": 0.0032933237962424755, + "policy/entropy_avg": 0.17766380310058594, + "step": 1013, + "val/clipfrac_avg": 0.006926534231752157, + "val/num_eos_tokens": 294, + "val/ratio": 0.987805962562561, + "val/ratio_var": 6.278733781073242e-05 + }, + { + "episode": 16224, + "epoch": 0.8193939393939393, + "eps": 0, + "loss/policy_avg": 0.06992897391319275, + "loss/value_avg": 0.47664809226989746, + "lr": 8.76345189483222e-07, + "objective/entropy": -88.55311584472656, + "objective/kl": 30.213661193847656, + "objective/non_score_reward": -1.5106831789016724, + "objective/rlhf_reward": 13.747129440307617, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007972473395057023, + "policy/clipfrac_avg": 0.00806876365095377, + "policy/entropy_avg": 0.1417522430419922, + "step": 1014, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 51, + "val/ratio": 0.9906024932861328, + "val/ratio_var": 5.377350316848606e-05 + }, + { + "episode": 16240, + "epoch": 0.8202020202020202, + "eps": 0, + "loss/policy_avg": 0.11788127571344376, + "loss/value_avg": 0.528445839881897, + "lr": 8.688061284200266e-07, + "objective/entropy": -92.63037109375, + "objective/kl": 36.9803466796875, + "objective/non_score_reward": -1.84901762008667, + "objective/rlhf_reward": 14.065045356750488, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008528477046638727, + "policy/clipfrac_avg": 0.0033233293797820807, + "policy/entropy_avg": 0.17418956756591797, + "step": 1015, + "val/clipfrac_avg": 0.013305861502885818, + "val/num_eos_tokens": 387, + "val/ratio": 0.989635169506073, + "val/ratio_var": 4.9907346692634746e-05 + }, + { + "episode": 16256, + "epoch": 0.821010101010101, + "eps": 0, + "loss/policy_avg": -0.04949585720896721, + "loss/value_avg": 0.3462344706058502, + "lr": 8.61296549592941e-07, + "objective/entropy": -62.09973907470703, + "objective/kl": 44.77581024169922, + "objective/non_score_reward": -2.23879075050354, + "objective/rlhf_reward": 12.683084487915039, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.001046380726620555, + "policy/clipfrac_avg": 0.00542850699275732, + "policy/entropy_avg": 0.2170124053955078, + "step": 1016, + "val/clipfrac_avg": 0.03059275820851326, + "val/num_eos_tokens": 270, + "val/ratio": 0.9858708381652832, + "val/ratio_var": 6.539487367263064e-05 + }, + { + "episode": 16272, + "epoch": 0.8218181818181818, + "eps": 0, + "loss/policy_avg": 0.03657815232872963, + "loss/value_avg": 0.687595546245575, + "lr": 8.538165065940263e-07, + "objective/entropy": -65.8448486328125, + "objective/kl": 34.462608337402344, + "objective/non_score_reward": -1.7231305837631226, + "objective/rlhf_reward": 13.925307273864746, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000925581669434905, + "policy/clipfrac_avg": 0.00542820617556572, + "policy/entropy_avg": 0.19491100311279297, + "step": 1017, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 15, + "val/ratio": 0.9877764582633972, + "val/ratio_var": 6.733841291861609e-05 + }, + { + "episode": 16288, + "epoch": 0.8226262626262626, + "eps": 0, + "loss/policy_avg": -0.097564198076725, + "loss/value_avg": 0.5997859835624695, + "lr": 8.463660528045514e-07, + "objective/entropy": -21.665847778320312, + "objective/kl": 41.799583435058594, + "objective/non_score_reward": -2.0899791717529297, + "objective/rlhf_reward": 14.79283332824707, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010365031193941832, + "policy/clipfrac_avg": 0.00511179631575942, + "policy/entropy_avg": 0.20884323120117188, + "step": 1018, + "val/clipfrac_avg": 0.008413461968302727, + "val/num_eos_tokens": 123, + "val/ratio": 0.9861449003219604, + "val/ratio_var": 5.511847120942548e-05 + }, + { + "episode": 16304, + "epoch": 0.8234343434343434, + "eps": 0, + "loss/policy_avg": -0.06873838603496552, + "loss/value_avg": 0.2803807854652405, + "lr": 8.389452413946314e-07, + "objective/entropy": -106.4912109375, + "objective/kl": 40.70600891113281, + "objective/non_score_reward": -2.0353002548217773, + "objective/rlhf_reward": 13.745949745178223, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008316466701216996, + "policy/clipfrac_avg": 0.0057511827908456326, + "policy/entropy_avg": 0.1492767333984375, + "step": 1019, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 205, + "val/ratio": 0.9896730184555054, + "val/ratio_var": 4.522735616774298e-05 + }, + { + "episode": 16320, + "epoch": 0.8242424242424242, + "eps": 0, + "loss/policy_avg": 0.018638040870428085, + "loss/value_avg": 0.7235357761383057, + "lr": 8.315541253228332e-07, + "objective/entropy": -29.912065505981445, + "objective/kl": 32.72511291503906, + "objective/non_score_reward": -1.6362557411193848, + "objective/rlhf_reward": 13.707494735717773, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0009581202175468206, + "policy/clipfrac_avg": 0.0044068810530006886, + "policy/entropy_avg": 0.2027587890625, + "step": 1020, + "val/clipfrac_avg": 0.009765625, + "val/num_eos_tokens": 35, + "val/ratio": 0.9886679649353027, + "val/ratio_var": 7.175742939580232e-05 + }, + { + "episode": 16336, + "epoch": 0.825050505050505, + "eps": 0, + "loss/policy_avg": 0.15834374725818634, + "loss/value_avg": 0.9719556570053101, + "lr": 8.241927573358078e-07, + "objective/entropy": -17.822988510131836, + "objective/kl": 44.23792266845703, + "objective/non_score_reward": -2.2118961811065674, + "objective/rlhf_reward": 13.405291557312012, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000986535451374948, + "policy/clipfrac_avg": 0.0043609049171209335, + "policy/entropy_avg": 0.19144058227539062, + "step": 1021, + "val/clipfrac_avg": 0.009959635324776173, + "val/num_eos_tokens": 44, + "val/ratio": 0.9864338040351868, + "val/ratio_var": 2.8990581995458342e-05 + }, + { + "episode": 16352, + "epoch": 0.8258585858585858, + "eps": 0, + "loss/policy_avg": 0.08962170779705048, + "loss/value_avg": 0.22370219230651855, + "lr": 8.168611899679013e-07, + "objective/entropy": -87.69083404541016, + "objective/kl": 41.61705017089844, + "objective/non_score_reward": -2.080852508544922, + "objective/rlhf_reward": 14.262897491455078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008991376380436122, + "policy/clipfrac_avg": 0.004512319341301918, + "policy/entropy_avg": 0.190032958984375, + "step": 1022, + "val/clipfrac_avg": 0.04466811940073967, + "val/num_eos_tokens": 17, + "val/ratio": 0.9874501824378967, + "val/ratio_var": 6.216822657734156e-05 + }, + { + "episode": 16368, + "epoch": 0.8266666666666667, + "eps": 0, + "loss/policy_avg": -0.6490851044654846, + "loss/value_avg": 0.6145610213279724, + "lr": 8.095594755407971e-07, + "objective/entropy": -103.04541778564453, + "objective/kl": 37.85612487792969, + "objective/non_score_reward": -1.8928062915802002, + "objective/rlhf_reward": 14.255630493164062, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009121837792918086, + "policy/clipfrac_avg": 0.004552682861685753, + "policy/entropy_avg": 0.17924374341964722, + "step": 1023, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 624, + "val/ratio": 0.9885929822921753, + "val/ratio_var": 8.515298395650461e-05 + }, + { + "episode": 16384, + "epoch": 0.8274747474747475, + "eps": 0, + "loss/policy_avg": 0.09050679206848145, + "loss/value_avg": 1.2950150966644287, + "lr": 8.022876661631268e-07, + "objective/entropy": -46.773006439208984, + "objective/kl": 33.627471923828125, + "objective/non_score_reward": -1.6813737154006958, + "objective/rlhf_reward": 14.521751403808594, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009958728915080428, + "policy/clipfrac_avg": 0.0035771888215094805, + "policy/entropy_avg": 0.17354488372802734, + "step": 1024, + "val/clipfrac_avg": 0.0034391595982015133, + "val/num_eos_tokens": 5, + "val/ratio": 0.9874868392944336, + "val/ratio_var": 7.542264211224392e-05 + }, + { + "episode": 16400, + "epoch": 0.8282828282828283, + "eps": 0, + "loss/policy_avg": 0.09637929499149323, + "loss/value_avg": 0.3471735417842865, + "lr": 7.950458137301082e-07, + "objective/entropy": -118.60977172851562, + "objective/kl": 40.922630310058594, + "objective/non_score_reward": -2.0461313724517822, + "objective/rlhf_reward": 14.555431365966797, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007377723813988268, + "policy/clipfrac_avg": 0.004692882765084505, + "policy/entropy_avg": 0.1540050506591797, + "step": 1025, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 445, + "val/ratio": 0.9898372888565063, + "val/ratio_var": 4.050094139529392e-05 + }, + { + "episode": 16416, + "epoch": 0.8290909090909091, + "eps": 0, + "loss/policy_avg": 0.003938224166631699, + "loss/value_avg": 0.12186452746391296, + "lr": 7.878339699231702e-07, + "objective/entropy": 5.97334623336792, + "objective/kl": 47.65478515625, + "objective/non_score_reward": -2.382739543914795, + "objective/rlhf_reward": 15.882884979248047, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011957846581935883, + "policy/clipfrac_avg": 0.0038270659279078245, + "policy/entropy_avg": 0.24349212646484375, + "step": 1026, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 63, + "val/ratio": 0.9836880564689636, + "val/ratio_var": 3.132951314910315e-05 + }, + { + "episode": 16432, + "epoch": 0.8298989898989899, + "eps": 0, + "loss/policy_avg": -0.008172400295734406, + "loss/value_avg": 0.40194565057754517, + "lr": 7.806521862095834e-07, + "objective/entropy": -145.87228393554688, + "objective/kl": 43.73109817504883, + "objective/non_score_reward": -2.1865549087524414, + "objective/rlhf_reward": 12.707976341247559, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000986664555966854, + "policy/clipfrac_avg": 0.005425687879323959, + "policy/entropy_avg": 0.21741676330566406, + "step": 1027, + "val/clipfrac_avg": 0.00168609619140625, + "val/num_eos_tokens": 2002, + "val/ratio": 0.9879361987113953, + "val/ratio_var": 7.765040209051222e-05 + }, + { + "episode": 16448, + "epoch": 0.8307070707070707, + "eps": 0, + "loss/policy_avg": -0.040230363607406616, + "loss/value_avg": 0.2879355549812317, + "lr": 7.735005138421003e-07, + "objective/entropy": -39.62914276123047, + "objective/kl": 30.83833122253418, + "objective/non_score_reward": -1.5419166088104248, + "objective/rlhf_reward": 13.919020652770996, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008982427534647286, + "policy/clipfrac_avg": 0.004154416732490063, + "policy/entropy_avg": 0.1513214111328125, + "step": 1028, + "val/clipfrac_avg": 0.0003633720916695893, + "val/num_eos_tokens": 42, + "val/ratio": 0.9900394678115845, + "val/ratio_var": 4.5976830733707175e-05 + }, + { + "episode": 16464, + "epoch": 0.8315151515151515, + "eps": 0, + "loss/policy_avg": -0.17712682485580444, + "loss/value_avg": 0.41484764218330383, + "lr": 7.663790038585794e-07, + "objective/entropy": -91.81086730957031, + "objective/kl": 39.669403076171875, + "objective/non_score_reward": -1.9834702014923096, + "objective/rlhf_reward": 14.73528003692627, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009472984238527715, + "policy/clipfrac_avg": 0.0033632363192737103, + "policy/entropy_avg": 0.1894826889038086, + "step": 1029, + "val/clipfrac_avg": 0.015109826810657978, + "val/num_eos_tokens": 637, + "val/ratio": 0.9873372316360474, + "val/ratio_var": 6.729063170496374e-05 + }, + { + "episode": 16480, + "epoch": 0.8323232323232324, + "eps": 0, + "loss/policy_avg": 0.03287976607680321, + "loss/value_avg": 0.3657122254371643, + "lr": 7.592877070816268e-07, + "objective/entropy": -81.92291259765625, + "objective/kl": 40.63905334472656, + "objective/non_score_reward": -2.0319526195526123, + "objective/rlhf_reward": 14.960234642028809, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009109035599976778, + "policy/clipfrac_avg": 0.003938743844628334, + "policy/entropy_avg": 0.17749857902526855, + "step": 1030, + "val/clipfrac_avg": 0.005359000526368618, + "val/num_eos_tokens": 56, + "val/ratio": 0.9891733527183533, + "val/ratio_var": 5.9037287428509444e-05 + }, + { + "episode": 16496, + "epoch": 0.8331313131313132, + "eps": 0, + "loss/policy_avg": 0.054874081164598465, + "loss/value_avg": 0.28462013602256775, + "lr": 7.522266741182305e-07, + "objective/entropy": -79.7171630859375, + "objective/kl": 43.41844177246094, + "objective/non_score_reward": -2.170922040939331, + "objective/rlhf_reward": 13.88376522064209, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009897679556161165, + "policy/clipfrac_avg": 0.00460906233638525, + "policy/entropy_avg": 0.18915176391601562, + "step": 1031, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 413, + "val/ratio": 0.987737238407135, + "val/ratio_var": 6.536820001201704e-05 + }, + { + "episode": 16512, + "epoch": 0.833939393939394, + "eps": 0, + "loss/policy_avg": 0.42096102237701416, + "loss/value_avg": 0.7095996141433716, + "lr": 7.451959553594051e-07, + "objective/entropy": -75.63118743896484, + "objective/kl": 32.546966552734375, + "objective/non_score_reward": -1.6273484230041504, + "objective/rlhf_reward": 12.075777053833008, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.000955111114308238, + "policy/clipfrac_avg": 0.005775670520961285, + "policy/entropy_avg": 0.16732215881347656, + "step": 1032, + "val/clipfrac_avg": 0.017921777442097664, + "val/num_eos_tokens": 145, + "val/ratio": 0.989020586013794, + "val/ratio_var": 4.018365507363342e-05 + }, + { + "episode": 16528, + "epoch": 0.8347474747474748, + "eps": 0, + "loss/policy_avg": 0.017501141875982285, + "loss/value_avg": 0.4045110046863556, + "lr": 7.381956009798253e-07, + "objective/entropy": -53.61748123168945, + "objective/kl": 38.599403381347656, + "objective/non_score_reward": -1.929970145225525, + "objective/rlhf_reward": 12.640341758728027, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008390340954065323, + "policy/clipfrac_avg": 0.0041229561902582645, + "policy/entropy_avg": 0.18261051177978516, + "step": 1033, + "val/clipfrac_avg": 0.020542385056614876, + "val/num_eos_tokens": 139, + "val/ratio": 0.988516092300415, + "val/ratio_var": 9.491335367783904e-05 + }, + { + "episode": 16544, + "epoch": 0.8355555555555556, + "eps": 0, + "loss/policy_avg": -0.03480757772922516, + "loss/value_avg": 1.401247501373291, + "lr": 7.312256609374708e-07, + "objective/entropy": -103.81524658203125, + "objective/kl": 36.218544006347656, + "objective/non_score_reward": -1.810927152633667, + "objective/rlhf_reward": 14.07969856262207, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007683830335736275, + "policy/clipfrac_avg": 0.003070900682359934, + "policy/entropy_avg": 0.14527511596679688, + "step": 1034, + "val/clipfrac_avg": 0.03183094039559364, + "val/num_eos_tokens": 65, + "val/ratio": 0.9899626970291138, + "val/ratio_var": 6.579139153473079e-05 + }, + { + "episode": 16560, + "epoch": 0.8363636363636363, + "eps": 0, + "loss/policy_avg": -0.28031888604164124, + "loss/value_avg": 1.1235523223876953, + "lr": 7.242861849732696e-07, + "objective/entropy": -59.279266357421875, + "objective/kl": 38.71485900878906, + "objective/non_score_reward": -1.935742974281311, + "objective/rlhf_reward": 14.07206916809082, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007580246892757714, + "policy/clipfrac_avg": 0.003035132307559252, + "policy/entropy_avg": 0.14683592319488525, + "step": 1035, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 109, + "val/ratio": 0.9899235963821411, + "val/ratio_var": 5.906899241381325e-05 + }, + { + "episode": 16576, + "epoch": 0.8371717171717171, + "eps": 0, + "loss/policy_avg": -0.14042839407920837, + "loss/value_avg": 1.1183263063430786, + "lr": 7.173772226107434e-07, + "objective/entropy": -64.92012023925781, + "objective/kl": 38.85520553588867, + "objective/non_score_reward": -1.9427603483200073, + "objective/rlhf_reward": 14.400989532470703, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009576823795214295, + "policy/clipfrac_avg": 0.004880891181528568, + "policy/entropy_avg": 0.19240808486938477, + "step": 1036, + "val/clipfrac_avg": 0.012853971682488918, + "val/num_eos_tokens": 155, + "val/ratio": 0.987326979637146, + "val/ratio_var": 9.263003448722884e-05 + }, + { + "episode": 16592, + "epoch": 0.8379797979797979, + "eps": 0, + "loss/policy_avg": -0.03311082720756531, + "loss/value_avg": 0.3677228093147278, + "lr": 7.104988231556543e-07, + "objective/entropy": -48.31360626220703, + "objective/kl": 36.47294616699219, + "objective/non_score_reward": -1.8236472606658936, + "objective/rlhf_reward": 14.137290000915527, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008876229985617101, + "policy/clipfrac_avg": 0.002901179250329733, + "policy/entropy_avg": 0.20080184936523438, + "step": 1037, + "val/clipfrac_avg": 0.002261103130877018, + "val/num_eos_tokens": 20, + "val/ratio": 0.9893977642059326, + "val/ratio_var": 5.7600576838012785e-05 + }, + { + "episode": 16608, + "epoch": 0.8387878787878787, + "eps": 0, + "loss/policy_avg": -0.10436652600765228, + "loss/value_avg": 0.43171289563179016, + "lr": 7.036510356956494e-07, + "objective/entropy": -66.36492919921875, + "objective/kl": 41.474632263183594, + "objective/non_score_reward": -2.0737314224243164, + "objective/rlhf_reward": 15.168456077575684, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009756431682035327, + "policy/clipfrac_avg": 0.005543471779674292, + "policy/entropy_avg": 0.188018798828125, + "step": 1038, + "val/clipfrac_avg": 0.01755008101463318, + "val/num_eos_tokens": 3, + "val/ratio": 0.9874497652053833, + "val/ratio_var": 5.146233888808638e-05 + }, + { + "episode": 16624, + "epoch": 0.8395959595959596, + "eps": 0, + "loss/policy_avg": -0.23364350199699402, + "loss/value_avg": 0.3087168335914612, + "lr": 6.968339090999188e-07, + "objective/entropy": -82.42080688476562, + "objective/kl": 33.46930694580078, + "objective/non_score_reward": -1.6734652519226074, + "objective/rlhf_reward": 16.849971771240234, + "objective/scores": 18.5, + "policy/approxkl_avg": 0.0009166139643639326, + "policy/clipfrac_avg": 0.0028331982903182507, + "policy/entropy_avg": 0.18764019012451172, + "step": 1039, + "val/clipfrac_avg": 0.022823384031653404, + "val/num_eos_tokens": 150, + "val/ratio": 0.9876014590263367, + "val/ratio_var": 7.868735701777041e-05 + }, + { + "episode": 16640, + "epoch": 0.8404040404040404, + "eps": 0, + "loss/policy_avg": -0.24348971247673035, + "loss/value_avg": 0.46729177236557007, + "lr": 6.900474920188383e-07, + "objective/entropy": -98.47803497314453, + "objective/kl": 31.56005096435547, + "objective/non_score_reward": -1.5780024528503418, + "objective/rlhf_reward": 12.9376220703125, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0007581596728414297, + "policy/clipfrac_avg": 0.0029252313543111086, + "policy/entropy_avg": 0.14857721328735352, + "step": 1040, + "val/clipfrac_avg": 0.018430940806865692, + "val/num_eos_tokens": 64, + "val/ratio": 0.9900062084197998, + "val/ratio_var": 7.637133239768445e-05 + }, + { + "episode": 16656, + "epoch": 0.8412121212121212, + "eps": 0, + "loss/policy_avg": -0.3006308078765869, + "loss/value_avg": 0.9436385035514832, + "lr": 6.832918328836247e-07, + "objective/entropy": -67.75382995605469, + "objective/kl": 32.892269134521484, + "objective/non_score_reward": -1.64461350440979, + "objective/rlhf_reward": 14.644449234008789, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009258019272238016, + "policy/clipfrac_avg": 0.004761096090078354, + "policy/entropy_avg": 0.1688389778137207, + "step": 1041, + "val/clipfrac_avg": 0.018809743225574493, + "val/num_eos_tokens": 357, + "val/ratio": 0.9893865585327148, + "val/ratio_var": 6.368513277266175e-05 + }, + { + "episode": 16672, + "epoch": 0.842020202020202, + "eps": 0, + "loss/policy_avg": -0.06437136977910995, + "loss/value_avg": 0.4288250207901001, + "lr": 6.765669799059915e-07, + "objective/entropy": -74.0018310546875, + "objective/kl": 35.5787467956543, + "objective/non_score_reward": -1.7789373397827148, + "objective/rlhf_reward": 13.291374206542969, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008827393758110702, + "policy/clipfrac_avg": 0.004510640166699886, + "policy/entropy_avg": 0.174835205078125, + "step": 1042, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 88, + "val/ratio": 0.9889439344406128, + "val/ratio_var": 5.196507117943838e-05 + }, + { + "episode": 16688, + "epoch": 0.8428282828282828, + "eps": 0, + "loss/policy_avg": 0.08304592221975327, + "loss/value_avg": 0.627299427986145, + "lr": 6.698729810778065e-07, + "objective/entropy": -94.1718521118164, + "objective/kl": 36.0411491394043, + "objective/non_score_reward": -1.8020575046539307, + "objective/rlhf_reward": 14.455755233764648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007288270280696452, + "policy/clipfrac_avg": 0.003802281804382801, + "policy/entropy_avg": 0.14253592491149902, + "step": 1043, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 492, + "val/ratio": 0.9900709390640259, + "val/ratio_var": 3.72955619241111e-05 + }, + { + "episode": 16704, + "epoch": 0.8436363636363636, + "eps": 0, + "loss/policy_avg": 0.289115309715271, + "loss/value_avg": 0.6000362038612366, + "lr": 6.632098841707458e-07, + "objective/entropy": -44.56480407714844, + "objective/kl": 37.80200958251953, + "objective/non_score_reward": -1.8901004791259766, + "objective/rlhf_reward": 13.203649520874023, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0012037131236866117, + "policy/clipfrac_avg": 0.005290274508297443, + "policy/entropy_avg": 0.21977996826171875, + "step": 1044, + "val/clipfrac_avg": 0.0035122863482683897, + "val/num_eos_tokens": 378, + "val/ratio": 0.9847176671028137, + "val/ratio_var": 9.624927770346403e-05 + }, + { + "episode": 16720, + "epoch": 0.8444444444444444, + "eps": 0, + "loss/policy_avg": 0.07545819133520126, + "loss/value_avg": 0.46948039531707764, + "lr": 6.565777367359516e-07, + "objective/entropy": -68.83721923828125, + "objective/kl": 39.69865798950195, + "objective/non_score_reward": -1.9849328994750977, + "objective/rlhf_reward": 13.468192100524902, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008786184480413795, + "policy/clipfrac_avg": 0.004983194172382355, + "policy/entropy_avg": 0.17647552490234375, + "step": 1045, + "val/clipfrac_avg": 0.00048828125, + "val/num_eos_tokens": 99, + "val/ratio": 0.9880126714706421, + "val/ratio_var": 4.486176840146072e-05 + }, + { + "episode": 16736, + "epoch": 0.8452525252525253, + "eps": 0, + "loss/policy_avg": 0.18258720636367798, + "loss/value_avg": 0.33022794127464294, + "lr": 6.499765861036994e-07, + "objective/entropy": -133.97775268554688, + "objective/kl": 39.10038375854492, + "objective/non_score_reward": -1.9550191164016724, + "objective/rlhf_reward": 13.568418502807617, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007223144639283419, + "policy/clipfrac_avg": 0.0023505911231040955, + "policy/entropy_avg": 0.1742839813232422, + "step": 1046, + "val/clipfrac_avg": 0.001006610575132072, + "val/num_eos_tokens": 767, + "val/ratio": 0.9903188943862915, + "val/ratio_var": 4.2478550312807783e-05 + }, + { + "episode": 16752, + "epoch": 0.8460606060606061, + "eps": 0, + "loss/policy_avg": 0.004041165113449097, + "loss/value_avg": 0.2533167600631714, + "lr": 6.43406479383053e-07, + "objective/entropy": -11.881628036499023, + "objective/kl": 40.72428894042969, + "objective/non_score_reward": -2.036214590072632, + "objective/rlhf_reward": 15.112222671508789, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0010564490221440792, + "policy/clipfrac_avg": 0.007933112792670727, + "policy/entropy_avg": 0.20357513427734375, + "step": 1047, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.9868043661117554, + "val/ratio_var": 6.374649819917977e-05 + }, + { + "episode": 16768, + "epoch": 0.8468686868686869, + "eps": 0, + "loss/policy_avg": 0.0763624757528305, + "loss/value_avg": 0.4388294219970703, + "lr": 6.368674634615357e-07, + "objective/entropy": -45.377681732177734, + "objective/kl": 34.73613739013672, + "objective/non_score_reward": -1.736806869506836, + "objective/rlhf_reward": 14.192880630493164, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009812908247113228, + "policy/clipfrac_avg": 0.0049714683555066586, + "policy/entropy_avg": 0.18614768981933594, + "step": 1048, + "val/clipfrac_avg": 0.0006127451197244227, + "val/num_eos_tokens": 351, + "val/ratio": 0.98753422498703, + "val/ratio_var": 4.261466165189631e-05 + }, + { + "episode": 16784, + "epoch": 0.8476767676767677, + "eps": 0, + "loss/policy_avg": -0.12992079555988312, + "loss/value_avg": 0.939970850944519, + "lr": 6.303595850047844e-07, + "objective/entropy": -80.03951263427734, + "objective/kl": 42.18169021606445, + "objective/non_score_reward": -2.1090846061706543, + "objective/rlhf_reward": 13.812790870666504, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009452086524106562, + "policy/clipfrac_avg": 0.004814188927412033, + "policy/entropy_avg": 0.1846628189086914, + "step": 1049, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 193, + "val/ratio": 0.988731324672699, + "val/ratio_var": 6.252430466702208e-05 + }, + { + "episode": 16800, + "epoch": 0.8484848484848485, + "eps": 0, + "loss/policy_avg": -0.12597322463989258, + "loss/value_avg": 0.5964376926422119, + "lr": 6.238828904562316e-07, + "objective/entropy": -56.887001037597656, + "objective/kl": 37.25836944580078, + "objective/non_score_reward": -1.8629186153411865, + "objective/rlhf_reward": 14.301143646240234, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008069226169027388, + "policy/clipfrac_avg": 0.0021998940501362085, + "policy/entropy_avg": 0.17313003540039062, + "step": 1050, + "val/clipfrac_avg": 0.00014467592700384557, + "val/num_eos_tokens": 69, + "val/ratio": 0.9889218807220459, + "val/ratio_var": 4.452227949514054e-05 + }, + { + "episode": 16816, + "epoch": 0.8492929292929293, + "eps": 0, + "loss/policy_avg": -0.284543514251709, + "loss/value_avg": 0.4945632815361023, + "lr": 6.174374260367611e-07, + "objective/entropy": -162.38363647460938, + "objective/kl": 36.81440734863281, + "objective/non_score_reward": -1.8407204151153564, + "objective/rlhf_reward": 13.549904823303223, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0006882078596390784, + "policy/clipfrac_avg": 0.0029306234791874886, + "policy/entropy_avg": 0.13589084148406982, + "step": 1051, + "val/clipfrac_avg": 0.02376413717865944, + "val/num_eos_tokens": 810, + "val/ratio": 0.9912172555923462, + "val/ratio_var": 4.5012933696853e-05 + }, + { + "episode": 16832, + "epoch": 0.8501010101010101, + "eps": 0, + "loss/policy_avg": -0.011637904681265354, + "loss/value_avg": 0.556053638458252, + "lr": 6.110232377443864e-07, + "objective/entropy": -106.74056243896484, + "objective/kl": 35.95270538330078, + "objective/non_score_reward": -1.7976351976394653, + "objective/rlhf_reward": 16.46017837524414, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0009089771774597466, + "policy/clipfrac_avg": 0.005288749001920223, + "policy/entropy_avg": 0.18283462524414062, + "step": 1052, + "val/clipfrac_avg": 0.0026041667442768812, + "val/num_eos_tokens": 104, + "val/ratio": 0.9882605075836182, + "val/ratio_var": 7.140174420783296e-05 + }, + { + "episode": 16848, + "epoch": 0.850909090909091, + "eps": 0, + "loss/policy_avg": -0.09856721758842468, + "loss/value_avg": 0.43345025181770325, + "lr": 6.04640371353914e-07, + "objective/entropy": -75.6595458984375, + "objective/kl": 40.826602935791016, + "objective/non_score_reward": -2.041330337524414, + "objective/rlhf_reward": 13.857107162475586, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009147390373982489, + "policy/clipfrac_avg": 0.0046502938494086266, + "policy/entropy_avg": 0.21721649169921875, + "step": 1053, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 61, + "val/ratio": 0.9873005151748657, + "val/ratio_var": 7.706157339271158e-05 + }, + { + "episode": 16864, + "epoch": 0.8517171717171718, + "eps": 0, + "loss/policy_avg": 0.051140159368515015, + "loss/value_avg": 0.46211516857147217, + "lr": 5.98288872416628e-07, + "objective/entropy": -99.58453369140625, + "objective/kl": 38.763389587402344, + "objective/non_score_reward": -1.938169240951538, + "objective/rlhf_reward": 15.054018020629883, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008463884005323052, + "policy/clipfrac_avg": 0.0038104294799268246, + "policy/entropy_avg": 0.16744399070739746, + "step": 1054, + "val/clipfrac_avg": 0.0019132653251290321, + "val/num_eos_tokens": 520, + "val/ratio": 0.989348292350769, + "val/ratio_var": 8.285223157145083e-05 + }, + { + "episode": 16880, + "epoch": 0.8525252525252526, + "eps": 0, + "loss/policy_avg": -0.03345664590597153, + "loss/value_avg": 0.41552969813346863, + "lr": 5.919687862599549e-07, + "objective/entropy": -79.0547103881836, + "objective/kl": 37.5797004699707, + "objective/non_score_reward": -1.878985047340393, + "objective/rlhf_reward": 14.566328048706055, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009661274962127209, + "policy/clipfrac_avg": 0.00524417357519269, + "policy/entropy_avg": 0.186629980802536, + "step": 1055, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 305, + "val/ratio": 0.9881995916366577, + "val/ratio_var": 7.692445797147229e-05 + }, + { + "episode": 16896, + "epoch": 0.8533333333333334, + "eps": 0, + "loss/policy_avg": 0.1846567541360855, + "loss/value_avg": 0.5820722579956055, + "lr": 5.856801579871457e-07, + "objective/entropy": -91.50765991210938, + "objective/kl": 38.447975158691406, + "objective/non_score_reward": -1.9223988056182861, + "objective/rlhf_reward": 12.522913932800293, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0007886831881478429, + "policy/clipfrac_avg": 0.00426090694963932, + "policy/entropy_avg": 0.15495681762695312, + "step": 1056, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 137, + "val/ratio": 0.9909533262252808, + "val/ratio_var": 5.192918251850642e-05 + }, + { + "episode": 16912, + "epoch": 0.8541414141414141, + "eps": 0, + "loss/policy_avg": -0.014410331845283508, + "loss/value_avg": 0.45579588413238525, + "lr": 5.794230324769518e-07, + "objective/entropy": -45.45872497558594, + "objective/kl": 41.44641876220703, + "objective/non_score_reward": -2.0723209381103516, + "objective/rlhf_reward": 13.966741561889648, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009499872685410082, + "policy/clipfrac_avg": 0.007878290489315987, + "policy/entropy_avg": 0.18343734741210938, + "step": 1057, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 399, + "val/ratio": 0.9887840747833252, + "val/ratio_var": 6.99822703609243e-05 + }, + { + "episode": 16928, + "epoch": 0.8549494949494949, + "eps": 0, + "loss/policy_avg": -0.05935902148485184, + "loss/value_avg": 0.36739227175712585, + "lr": 5.731974543833041e-07, + "objective/entropy": -55.36639404296875, + "objective/kl": 38.324073791503906, + "objective/non_score_reward": -1.916203498840332, + "objective/rlhf_reward": 12.029109001159668, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009155046427622437, + "policy/clipfrac_avg": 0.003906010650098324, + "policy/entropy_avg": 0.17214488983154297, + "step": 1058, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891675114631653, + "val/ratio_var": 5.269415851216763e-05 + }, + { + "episode": 16944, + "epoch": 0.8557575757575757, + "eps": 0, + "loss/policy_avg": 0.12509797513484955, + "loss/value_avg": 0.5901766419410706, + "lr": 5.670034681349995e-07, + "objective/entropy": -77.39466857910156, + "objective/kl": 39.27944564819336, + "objective/non_score_reward": -1.9639724493026733, + "objective/rlhf_reward": 12.512590408325195, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0011488801101222634, + "policy/clipfrac_avg": 0.005929230712354183, + "policy/entropy_avg": 0.22491836547851562, + "step": 1059, + "val/clipfrac_avg": 0.01009130198508501, + "val/num_eos_tokens": 274, + "val/ratio": 0.9847910404205322, + "val/ratio_var": 8.84500186657533e-05 + }, + { + "episode": 16960, + "epoch": 0.8565656565656565, + "eps": 0, + "loss/policy_avg": 0.027433840557932854, + "loss/value_avg": 0.2936461567878723, + "lr": 5.608411179353767e-07, + "objective/entropy": -68.70003509521484, + "objective/kl": 34.862098693847656, + "objective/non_score_reward": -1.7431049346923828, + "objective/rlhf_reward": 15.538145065307617, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010132534662261605, + "policy/clipfrac_avg": 0.00747870746999979, + "policy/entropy_avg": 0.19480419158935547, + "step": 1060, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 6, + "val/ratio": 0.9880098700523376, + "val/ratio_var": 6.574365397682413e-05 + }, + { + "episode": 16976, + "epoch": 0.8573737373737373, + "eps": 0, + "loss/policy_avg": 0.22168952226638794, + "loss/value_avg": 0.6544172763824463, + "lr": 5.547104477620041e-07, + "objective/entropy": -101.6231918334961, + "objective/kl": 36.893959045410156, + "objective/non_score_reward": -1.844698190689087, + "objective/rlhf_reward": 13.850614547729492, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008218783186748624, + "policy/clipfrac_avg": 0.005969468504190445, + "policy/entropy_avg": 0.15513205528259277, + "step": 1061, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 465, + "val/ratio": 0.9892902374267578, + "val/ratio_var": 6.282331742113456e-05 + }, + { + "episode": 16992, + "epoch": 0.8581818181818182, + "eps": 0, + "loss/policy_avg": -0.23859268426895142, + "loss/value_avg": 0.5853025913238525, + "lr": 5.486115013663668e-07, + "objective/entropy": -112.72584533691406, + "objective/kl": 33.110145568847656, + "objective/non_score_reward": -1.6555070877075195, + "objective/rlhf_reward": 15.28980541229248, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0006752893095836043, + "policy/clipfrac_avg": 0.0032210415229201317, + "policy/entropy_avg": 0.12448978424072266, + "step": 1062, + "val/clipfrac_avg": 4.234417428961024e-05, + "val/num_eos_tokens": 451, + "val/ratio": 0.9915745258331299, + "val/ratio_var": 5.202127795200795e-05 + }, + { + "episode": 17008, + "epoch": 0.858989898989899, + "eps": 0, + "loss/policy_avg": -0.1410359889268875, + "loss/value_avg": 0.22414374351501465, + "lr": 5.425443222735527e-07, + "objective/entropy": -49.841190338134766, + "objective/kl": 38.634700775146484, + "objective/non_score_reward": -1.9317350387573242, + "objective/rlhf_reward": 16.25576400756836, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011730990372598171, + "policy/clipfrac_avg": 0.008051658049225807, + "policy/entropy_avg": 0.22179222106933594, + "step": 1063, + "val/clipfrac_avg": 0.009043844416737556, + "val/num_eos_tokens": 125, + "val/ratio": 0.9854654669761658, + "val/ratio_var": 7.81394264777191e-05 + }, + { + "episode": 17024, + "epoch": 0.8597979797979798, + "eps": 0, + "loss/policy_avg": -0.060928136110305786, + "loss/value_avg": 0.45962244272232056, + "lr": 5.365089537819435e-07, + "objective/entropy": -68.24198913574219, + "objective/kl": 37.95829772949219, + "objective/non_score_reward": -1.8979148864746094, + "objective/rlhf_reward": 15.38333511352539, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008637227001599967, + "policy/clipfrac_avg": 0.006955782417207956, + "policy/entropy_avg": 0.1758570671081543, + "step": 1064, + "val/clipfrac_avg": 0.027727147564291954, + "val/num_eos_tokens": 423, + "val/ratio": 0.9886413812637329, + "val/ratio_var": 8.117071411106735e-05 + }, + { + "episode": 17040, + "epoch": 0.8606060606060606, + "eps": 0, + "loss/policy_avg": -0.024640943855047226, + "loss/value_avg": 0.736940324306488, + "lr": 5.305054389629022e-07, + "objective/entropy": -66.94715881347656, + "objective/kl": 41.044742584228516, + "objective/non_score_reward": -2.052237033843994, + "objective/rlhf_reward": 13.018075942993164, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009623835212551057, + "policy/clipfrac_avg": 0.0037963923532515764, + "policy/entropy_avg": 0.1932220458984375, + "step": 1065, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 86, + "val/ratio": 0.9877476692199707, + "val/ratio_var": 5.7415742048760876e-05 + }, + { + "episode": 17056, + "epoch": 0.8614141414141414, + "eps": 0, + "loss/policy_avg": 0.11599922180175781, + "loss/value_avg": 1.5807065963745117, + "lr": 5.245338206604733e-07, + "objective/entropy": -115.46347045898438, + "objective/kl": 40.31273651123047, + "objective/non_score_reward": -2.015636682510376, + "objective/rlhf_reward": 13.367176055908203, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008412263705395162, + "policy/clipfrac_avg": 0.01834527961909771, + "policy/entropy_avg": 0.17316818237304688, + "step": 1066, + "val/clipfrac_avg": 1.0643732821336016e-05, + "val/num_eos_tokens": 71, + "val/ratio": 0.9887286424636841, + "val/ratio_var": 4.431970592122525e-05 + }, + { + "episode": 17072, + "epoch": 0.8622222222222222, + "eps": 0, + "loss/policy_avg": -0.11949857324361801, + "loss/value_avg": 0.5354464650154114, + "lr": 5.185941414910673e-07, + "objective/entropy": -67.14189147949219, + "objective/kl": 39.36028289794922, + "objective/non_score_reward": -1.9680140018463135, + "objective/rlhf_reward": 15.414798736572266, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0008963524596765637, + "policy/clipfrac_avg": 0.00728583661839366, + "policy/entropy_avg": 0.17730861902236938, + "step": 1067, + "val/clipfrac_avg": 0.0072115384973585606, + "val/num_eos_tokens": 286, + "val/ratio": 0.9881836175918579, + "val/ratio_var": 3.5629829653771594e-05 + }, + { + "episode": 17088, + "epoch": 0.863030303030303, + "eps": 0, + "loss/policy_avg": 0.10199490189552307, + "loss/value_avg": 0.35868531465530396, + "lr": 5.126864438431628e-07, + "objective/entropy": -115.96305847167969, + "objective/kl": 37.194644927978516, + "objective/non_score_reward": -1.8597323894500732, + "objective/rlhf_reward": 12.882455825805664, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000633776537142694, + "policy/clipfrac_avg": 0.004217084962874651, + "policy/entropy_avg": 0.13162803649902344, + "step": 1068, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 156, + "val/ratio": 0.9914271235466003, + "val/ratio_var": 5.447517105494626e-05 + }, + { + "episode": 17104, + "epoch": 0.8638383838383838, + "eps": 0, + "loss/policy_avg": 0.09657607972621918, + "loss/value_avg": 0.33049535751342773, + "lr": 5.06810769877002e-07, + "objective/entropy": -27.455425262451172, + "objective/kl": 30.918790817260742, + "objective/non_score_reward": -1.545939564704895, + "objective/rlhf_reward": 13.969684600830078, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000932033930439502, + "policy/clipfrac_avg": 0.0023164437152445316, + "policy/entropy_avg": 0.2050342559814453, + "step": 1069, + "val/clipfrac_avg": 0.0017710273386910558, + "val/num_eos_tokens": 117, + "val/ratio": 0.9869778752326965, + "val/ratio_var": 7.318282587220892e-05 + }, + { + "episode": 17120, + "epoch": 0.8646464646464647, + "eps": 0, + "loss/policy_avg": -0.07673387229442596, + "loss/value_avg": 0.6429687738418579, + "lr": 5.009671615242928e-07, + "objective/entropy": -41.616695404052734, + "objective/kl": 42.76515197753906, + "objective/non_score_reward": -2.1382575035095215, + "objective/rlhf_reward": 13.17424201965332, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009901263983920217, + "policy/clipfrac_avg": 0.003959751222282648, + "policy/entropy_avg": 0.1980295181274414, + "step": 1070, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 199, + "val/ratio": 0.987491250038147, + "val/ratio_var": 4.666786844609305e-05 + }, + { + "episode": 17136, + "epoch": 0.8654545454545455, + "eps": 0, + "loss/policy_avg": -0.03734193369746208, + "loss/value_avg": 0.7440835237503052, + "lr": 4.951556604879049e-07, + "objective/entropy": -53.619712829589844, + "objective/kl": 39.59283447265625, + "objective/non_score_reward": -1.9796417951583862, + "objective/rlhf_reward": 13.887545585632324, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010703656589612365, + "policy/clipfrac_avg": 0.0033148752991110086, + "policy/entropy_avg": 0.19559097290039062, + "step": 1071, + "val/clipfrac_avg": 0.00032552084303461015, + "val/num_eos_tokens": 134, + "val/ratio": 0.9871270060539246, + "val/ratio_var": 6.861035944893956e-05 + }, + { + "episode": 17152, + "epoch": 0.8662626262626263, + "eps": 0, + "loss/policy_avg": 0.05410761386156082, + "loss/value_avg": 0.6214333772659302, + "lr": 4.893763082415736e-07, + "objective/entropy": -32.19466781616211, + "objective/kl": 33.41657638549805, + "objective/non_score_reward": -1.6708288192749023, + "objective/rlhf_reward": 12.516671180725098, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010530936997383833, + "policy/clipfrac_avg": 0.005051491782069206, + "policy/entropy_avg": 0.18621063232421875, + "step": 1072, + "val/clipfrac_avg": 0.006988893263041973, + "val/num_eos_tokens": 41, + "val/ratio": 0.9861816167831421, + "val/ratio_var": 5.813246752950363e-05 + }, + { + "episode": 17168, + "epoch": 0.8670707070707071, + "eps": 0, + "loss/policy_avg": -0.13285210728645325, + "loss/value_avg": 0.37682080268859863, + "lr": 4.836291460296066e-07, + "objective/entropy": -73.75119018554688, + "objective/kl": 44.275638580322266, + "objective/non_score_reward": -2.2137818336486816, + "objective/rlhf_reward": 13.872156143188477, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009736482752487063, + "policy/clipfrac_avg": 0.002118697389960289, + "policy/entropy_avg": 0.1989288330078125, + "step": 1073, + "val/clipfrac_avg": 0.003361742477864027, + "val/num_eos_tokens": 178, + "val/ratio": 0.9864859580993652, + "val/ratio_var": 7.720785652054474e-05 + }, + { + "episode": 17184, + "epoch": 0.8678787878787879, + "eps": 0, + "loss/policy_avg": 0.032950110733509064, + "loss/value_avg": 0.5699096918106079, + "lr": 4.779142148665855e-07, + "objective/entropy": -17.03137969970703, + "objective/kl": 29.793800354003906, + "objective/non_score_reward": -1.4896900653839111, + "objective/rlhf_reward": 15.447810173034668, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.000868711736984551, + "policy/clipfrac_avg": 0.0015887501649558544, + "policy/entropy_avg": 0.17066001892089844, + "step": 1074, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 84, + "val/ratio": 0.9895613193511963, + "val/ratio_var": 7.112583261914551e-05 + }, + { + "episode": 17200, + "epoch": 0.8686868686868687, + "eps": 0, + "loss/policy_avg": -0.11098457127809525, + "loss/value_avg": 0.5394589304924011, + "lr": 4.722315555370793e-07, + "objective/entropy": -46.34983825683594, + "objective/kl": 39.953025817871094, + "objective/non_score_reward": -1.997651219367981, + "objective/rlhf_reward": 14.674223899841309, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008599478751420975, + "policy/clipfrac_avg": 0.006935285869985819, + "policy/entropy_avg": 0.17142105102539062, + "step": 1075, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 73, + "val/ratio": 0.9890701770782471, + "val/ratio_var": 3.540362376952544e-05 + }, + { + "episode": 17216, + "epoch": 0.8694949494949495, + "eps": 0, + "loss/policy_avg": 0.11330074071884155, + "loss/value_avg": 0.47666221857070923, + "lr": 4.665812085953425e-07, + "objective/entropy": -62.084415435791016, + "objective/kl": 43.93402099609375, + "objective/non_score_reward": -2.1967010498046875, + "objective/rlhf_reward": 12.162673950195312, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009052088716998696, + "policy/clipfrac_avg": 0.0032377864699810743, + "policy/entropy_avg": 0.18408584594726562, + "step": 1076, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 588, + "val/ratio": 0.9876794815063477, + "val/ratio_var": 4.1342565964441746e-05 + }, + { + "episode": 17232, + "epoch": 0.8703030303030304, + "eps": 0, + "loss/policy_avg": -0.05885958671569824, + "loss/value_avg": 0.9560117125511169, + "lr": 4.6096321436504e-07, + "objective/entropy": -73.24485778808594, + "objective/kl": 39.06366729736328, + "objective/non_score_reward": -1.9531835317611694, + "objective/rlhf_reward": 13.5468168258667, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009686506818979979, + "policy/clipfrac_avg": 0.004011271521449089, + "policy/entropy_avg": 0.18913555145263672, + "step": 1077, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 200, + "val/ratio": 0.9870175719261169, + "val/ratio_var": 8.096115197986364e-05 + }, + { + "episode": 17248, + "epoch": 0.8711111111111111, + "eps": 0, + "loss/policy_avg": 0.18471674621105194, + "loss/value_avg": 0.37227392196655273, + "lr": 4.5537761293894535e-07, + "objective/entropy": -116.47470092773438, + "objective/kl": 37.137413024902344, + "objective/non_score_reward": -1.8568708896636963, + "objective/rlhf_reward": 14.861879348754883, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000727109843865037, + "policy/clipfrac_avg": 0.002056294586509466, + "policy/entropy_avg": 0.1612076759338379, + "step": 1078, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 178, + "val/ratio": 0.9902770519256592, + "val/ratio_var": 6.446443876484409e-05 + }, + { + "episode": 17264, + "epoch": 0.8719191919191919, + "eps": 0, + "loss/policy_avg": -0.16059936583042145, + "loss/value_avg": 0.507465124130249, + "lr": 4.4982444417866753e-07, + "objective/entropy": -93.82952880859375, + "objective/kl": 37.85641098022461, + "objective/non_score_reward": -1.8928205966949463, + "objective/rlhf_reward": 13.458742141723633, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0009164538932964206, + "policy/clipfrac_avg": 0.0050585465505719185, + "policy/entropy_avg": 0.15972977876663208, + "step": 1079, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 44, + "val/ratio": 0.989949643611908, + "val/ratio_var": 8.05418094387278e-05 + }, + { + "episode": 17280, + "epoch": 0.8727272727272727, + "eps": 0, + "loss/policy_avg": 0.024168267846107483, + "loss/value_avg": 0.5276412963867188, + "lr": 4.4430374771435245e-07, + "objective/entropy": -93.29814910888672, + "objective/kl": 36.883766174316406, + "objective/non_score_reward": -1.8441884517669678, + "objective/rlhf_reward": 13.952686309814453, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0010746137704700232, + "policy/clipfrac_avg": 0.005830212030559778, + "policy/entropy_avg": 0.1792011260986328, + "step": 1080, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 519, + "val/ratio": 0.989915132522583, + "val/ratio_var": 4.745582918985747e-05 + }, + { + "episode": 17296, + "epoch": 0.8735353535353535, + "eps": 0, + "loss/policy_avg": -0.09750235080718994, + "loss/value_avg": 0.4400123357772827, + "lr": 4.3881556294441516e-07, + "objective/entropy": -163.2279815673828, + "objective/kl": 42.45173645019531, + "objective/non_score_reward": -2.122586727142334, + "objective/rlhf_reward": 13.236787796020508, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007963902316987514, + "policy/clipfrac_avg": 0.0037931441329419613, + "policy/entropy_avg": 0.17099809646606445, + "step": 1081, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 294, + "val/ratio": 0.9886674880981445, + "val/ratio_var": 8.438028453383595e-05 + }, + { + "episode": 17312, + "epoch": 0.8743434343434343, + "eps": 0, + "loss/policy_avg": -0.0659959465265274, + "loss/value_avg": 0.2975969910621643, + "lr": 4.333599290352458e-07, + "objective/entropy": -63.06423568725586, + "objective/kl": 42.34510803222656, + "objective/non_score_reward": -2.117255210876465, + "objective/rlhf_reward": 14.288995742797852, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009070255910046399, + "policy/clipfrac_avg": 0.0022676163353025913, + "policy/entropy_avg": 0.18258285522460938, + "step": 1082, + "val/clipfrac_avg": 0.0035635964013636112, + "val/num_eos_tokens": 576, + "val/ratio": 0.9886539578437805, + "val/ratio_var": 6.365034641930833e-05 + }, + { + "episode": 17328, + "epoch": 0.8751515151515151, + "eps": 0, + "loss/policy_avg": -0.12663836777210236, + "loss/value_avg": 1.6143395900726318, + "lr": 4.279368849209381e-07, + "objective/entropy": -124.63575744628906, + "objective/kl": 34.96723937988281, + "objective/non_score_reward": -1.7483620643615723, + "objective/rlhf_reward": 15.900075912475586, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0008090031915344298, + "policy/clipfrac_avg": 0.0031590727157890797, + "policy/entropy_avg": 0.16525745391845703, + "step": 1083, + "val/clipfrac_avg": 0.02003205195069313, + "val/num_eos_tokens": 359, + "val/ratio": 0.9890646934509277, + "val/ratio_var": 7.246930908877403e-05 + }, + { + "episode": 17344, + "epoch": 0.8759595959595959, + "eps": 0, + "loss/policy_avg": 0.10138820856809616, + "loss/value_avg": 0.3533989191055298, + "lr": 4.2254646930300845e-07, + "objective/entropy": -55.43729019165039, + "objective/kl": 44.472023010253906, + "objective/non_score_reward": -2.2236011028289795, + "objective/rlhf_reward": 13.721712112426758, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009467223426327109, + "policy/clipfrac_avg": 0.0063713230192661285, + "policy/entropy_avg": 0.1949615478515625, + "step": 1084, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.986469030380249, + "val/ratio_var": 5.372995656216517e-05 + }, + { + "episode": 17360, + "epoch": 0.8767676767676768, + "eps": 0, + "loss/policy_avg": -0.16358020901679993, + "loss/value_avg": 0.7672687768936157, + "lr": 4.171887206501191e-07, + "objective/entropy": -80.793701171875, + "objective/kl": 35.82070541381836, + "objective/non_score_reward": -1.7910352945327759, + "objective/rlhf_reward": 13.552715301513672, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007973198080435395, + "policy/clipfrac_avg": 0.003301304765045643, + "policy/entropy_avg": 0.15000247955322266, + "step": 1085, + "val/clipfrac_avg": 0.0021186440717428923, + "val/num_eos_tokens": 403, + "val/ratio": 0.9896234273910522, + "val/ratio_var": 5.3207288146950305e-05 + }, + { + "episode": 17376, + "epoch": 0.8775757575757576, + "eps": 0, + "loss/policy_avg": 0.20687676966190338, + "loss/value_avg": 1.3663004636764526, + "lr": 4.1186367719780737e-07, + "objective/entropy": -46.82775115966797, + "objective/kl": 33.1654167175293, + "objective/non_score_reward": -1.6582709550857544, + "objective/rlhf_reward": 14.404230117797852, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008230544626712799, + "policy/clipfrac_avg": 0.0030164497438818216, + "policy/entropy_avg": 0.1555004119873047, + "step": 1086, + "val/clipfrac_avg": 0.0026466259732842445, + "val/num_eos_tokens": 47, + "val/ratio": 0.9892673492431641, + "val/ratio_var": 3.7006568163633347e-05 + }, + { + "episode": 17392, + "epoch": 0.8783838383838384, + "eps": 0, + "loss/policy_avg": 0.18984848260879517, + "loss/value_avg": 0.31801366806030273, + "lr": 4.0657137694820826e-07, + "objective/entropy": -66.93091583251953, + "objective/kl": 45.35851287841797, + "objective/non_score_reward": -2.26792573928833, + "objective/rlhf_reward": 14.122699737548828, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008988711051642895, + "policy/clipfrac_avg": 0.004365711007267237, + "policy/entropy_avg": 0.17530107498168945, + "step": 1087, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 363, + "val/ratio": 0.9879393577575684, + "val/ratio_var": 6.273078179219738e-05 + }, + { + "episode": 17408, + "epoch": 0.8791919191919192, + "eps": 0, + "loss/policy_avg": 0.14154401421546936, + "loss/value_avg": 0.296213299036026, + "lr": 4.013118576697844e-07, + "objective/entropy": -96.627197265625, + "objective/kl": 44.89594650268555, + "objective/non_score_reward": -2.244797468185425, + "objective/rlhf_reward": 14.028639793395996, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009503285982646048, + "policy/clipfrac_avg": 0.004713431932032108, + "policy/entropy_avg": 0.19094181060791016, + "step": 1088, + "val/clipfrac_avg": 0.004036702681332827, + "val/num_eos_tokens": 215, + "val/ratio": 0.9875110983848572, + "val/ratio_var": 3.645908509497531e-05 + }, + { + "episode": 17424, + "epoch": 0.88, + "eps": 0, + "loss/policy_avg": -0.2717769145965576, + "loss/value_avg": 0.6515541076660156, + "lr": 3.960851568970586e-07, + "objective/entropy": -81.71890258789062, + "objective/kl": 44.09088897705078, + "objective/non_score_reward": -2.2045445442199707, + "objective/rlhf_reward": 14.311080932617188, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008286006050184369, + "policy/clipfrac_avg": 0.0029842001385986805, + "policy/entropy_avg": 0.16189241409301758, + "step": 1089, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 391, + "val/ratio": 0.9892988204956055, + "val/ratio_var": 5.128037810209207e-05 + }, + { + "episode": 17440, + "epoch": 0.8808080808080808, + "eps": 0, + "loss/policy_avg": -0.09007427841424942, + "loss/value_avg": 0.174169659614563, + "lr": 3.908913119303448e-07, + "objective/entropy": -103.39527893066406, + "objective/kl": 45.33392333984375, + "objective/non_score_reward": -2.2666964530944824, + "objective/rlhf_reward": 15.030179023742676, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008385264663957059, + "policy/clipfrac_avg": 0.0022599862422794104, + "policy/entropy_avg": 0.167741060256958, + "step": 1090, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9883161783218384, + "val/ratio_var": 4.988587534171529e-05 + }, + { + "episode": 17456, + "epoch": 0.8816161616161616, + "eps": 0, + "loss/policy_avg": 0.29356396198272705, + "loss/value_avg": 0.8743624687194824, + "lr": 3.8573035983548167e-07, + "objective/entropy": -82.11065673828125, + "objective/kl": 38.23064422607422, + "objective/non_score_reward": -1.9115324020385742, + "objective/rlhf_reward": 13.299405097961426, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008716207230463624, + "policy/clipfrac_avg": 0.0072744423523545265, + "policy/entropy_avg": 0.16114044189453125, + "step": 1091, + "val/clipfrac_avg": 0.0008928571478463709, + "val/num_eos_tokens": 91, + "val/ratio": 0.9882928133010864, + "val/ratio_var": 6.514011329272762e-05 + }, + { + "episode": 17472, + "epoch": 0.8824242424242424, + "eps": 0, + "loss/policy_avg": 0.02994067780673504, + "loss/value_avg": 0.792471170425415, + "lr": 3.8060233744356634e-07, + "objective/entropy": -12.753183364868164, + "objective/kl": 37.99050521850586, + "objective/non_score_reward": -1.8995252847671509, + "objective/rlhf_reward": 12.65125560760498, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0010071105789393187, + "policy/clipfrac_avg": 0.004005424678325653, + "policy/entropy_avg": 0.19425201416015625, + "step": 1092, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 14, + "val/ratio": 0.9855411052703857, + "val/ratio_var": 4.6690693125128746e-05 + }, + { + "episode": 17488, + "epoch": 0.8832323232323233, + "eps": 0, + "loss/policy_avg": 0.1304478645324707, + "loss/value_avg": 0.3748246431350708, + "lr": 3.7550728135069545e-07, + "objective/entropy": -36.16312789916992, + "objective/kl": 48.05560302734375, + "objective/non_score_reward": -2.402780532836914, + "objective/rlhf_reward": 14.300344467163086, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000979696400463581, + "policy/clipfrac_avg": 0.005794232711195946, + "policy/entropy_avg": 0.19660568237304688, + "step": 1093, + "val/clipfrac_avg": 0.0026041667442768812, + "val/num_eos_tokens": 66, + "val/ratio": 0.9865772128105164, + "val/ratio_var": 5.429132215795107e-05 + }, + { + "episode": 17504, + "epoch": 0.8840404040404041, + "eps": 0, + "loss/policy_avg": 0.07484120875597, + "loss/value_avg": 0.2602160573005676, + "lr": 3.704452279177012e-07, + "objective/entropy": -71.43814086914062, + "objective/kl": 45.28865051269531, + "objective/non_score_reward": -2.264432668685913, + "objective/rlhf_reward": 13.727754592895508, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000950466375797987, + "policy/clipfrac_avg": 0.0027152306865900755, + "policy/entropy_avg": 0.20662879943847656, + "step": 1094, + "val/clipfrac_avg": 0.0006720430101267993, + "val/num_eos_tokens": 494, + "val/ratio": 0.9866384863853455, + "val/ratio_var": 7.357261347351596e-05 + }, + { + "episode": 17520, + "epoch": 0.8848484848484849, + "eps": 0, + "loss/policy_avg": 0.10963790118694305, + "loss/value_avg": 0.7320235967636108, + "lr": 3.6541621326989183e-07, + "objective/entropy": -48.0823974609375, + "objective/kl": 39.92009353637695, + "objective/non_score_reward": -1.9960048198699951, + "objective/rlhf_reward": 12.980557441711426, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0010038036853075027, + "policy/clipfrac_avg": 0.004399286117404699, + "policy/entropy_avg": 0.19763946533203125, + "step": 1095, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 436, + "val/ratio": 0.987025260925293, + "val/ratio_var": 3.8341207982739434e-05 + }, + { + "episode": 17536, + "epoch": 0.8856565656565657, + "eps": 0, + "loss/policy_avg": 0.05299535021185875, + "loss/value_avg": 0.2717387080192566, + "lr": 3.6042027329679487e-07, + "objective/entropy": -51.004615783691406, + "objective/kl": 41.74889373779297, + "objective/non_score_reward": -2.087444543838501, + "objective/rlhf_reward": 14.545368194580078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010824615601450205, + "policy/clipfrac_avg": 0.00963993463665247, + "policy/entropy_avg": 0.20147705078125, + "step": 1096, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 345, + "val/ratio": 0.9875917434692383, + "val/ratio_var": 6.199166091391817e-05 + }, + { + "episode": 17552, + "epoch": 0.8864646464646465, + "eps": 0, + "loss/policy_avg": -0.08447069674730301, + "loss/value_avg": 0.38099169731140137, + "lr": 3.5545744365190204e-07, + "objective/entropy": -108.7376937866211, + "objective/kl": 41.82084655761719, + "objective/non_score_reward": -2.0910425186157227, + "objective/rlhf_reward": 14.158958435058594, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000758812646381557, + "policy/clipfrac_avg": 0.003666321514174342, + "policy/entropy_avg": 0.15442466735839844, + "step": 1097, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 244, + "val/ratio": 0.9902445077896118, + "val/ratio_var": 2.8433618354029022e-05 + }, + { + "episode": 17568, + "epoch": 0.8872727272727273, + "eps": 0, + "loss/policy_avg": -0.47139230370521545, + "loss/value_avg": 1.4234567880630493, + "lr": 3.5052775975241203e-07, + "objective/entropy": -131.56527709960938, + "objective/kl": 33.65018081665039, + "objective/non_score_reward": -1.682509183883667, + "objective/rlhf_reward": 13.74717903137207, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009085559286177158, + "policy/clipfrac_avg": 0.004985325504094362, + "policy/entropy_avg": 0.16993939876556396, + "step": 1098, + "val/clipfrac_avg": 0.00024242099607363343, + "val/num_eos_tokens": 1220, + "val/ratio": 0.9888011813163757, + "val/ratio_var": 7.847072265576571e-05 + }, + { + "episode": 17584, + "epoch": 0.888080808080808, + "eps": 0, + "loss/policy_avg": 0.16145119071006775, + "loss/value_avg": 0.5360366106033325, + "lr": 3.4563125677897936e-07, + "objective/entropy": -42.34114074707031, + "objective/kl": 37.358299255371094, + "objective/non_score_reward": -1.867915153503418, + "objective/rlhf_reward": 14.428959846496582, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001106644980609417, + "policy/clipfrac_avg": 0.0073922015726566315, + "policy/entropy_avg": 0.216278076171875, + "step": 1099, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 172, + "val/ratio": 0.9858206510543823, + "val/ratio_var": 6.1017286498099566e-05 + }, + { + "episode": 17600, + "epoch": 0.8888888888888888, + "eps": 0, + "loss/policy_avg": 0.043675780296325684, + "loss/value_avg": 0.5687451362609863, + "lr": 3.4076796967546176e-07, + "objective/entropy": -32.0341682434082, + "objective/kl": 43.24127197265625, + "objective/non_score_reward": -2.1620638370513916, + "objective/rlhf_reward": 12.939498901367188, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010629122843965888, + "policy/clipfrac_avg": 0.003724365960806608, + "policy/entropy_avg": 0.20337867736816406, + "step": 1100, + "val/clipfrac_avg": 0.005150945391505957, + "val/num_eos_tokens": 106, + "val/ratio": 0.9857684969902039, + "val/ratio_var": 4.0027771319728345e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4c67e733836cc17326c1b8036582b07f2b472b6 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea3c5c45907eff697eca076ee5ead356e742227304ed0ab4edb73b0429b8fda +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2b530276f7c9887e068cbc532bf924ebf2052c4e --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/trainer_state.json @@ -0,0 +1,25235 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 19200, + "epoch": 0.9696969696969697, + "eval_steps": 100, + "global_step": 1200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + }, + { + "episode": 11216, + "epoch": 0.5664646464646464, + "eps": 0, + "loss/policy_avg": -0.09055965393781662, + "loss/value_avg": 0.5455071926116943, + "lr": 4.334128543718295e-06, + "objective/entropy": -74.95745849609375, + "objective/kl": 33.42132568359375, + "objective/non_score_reward": -1.6710662841796875, + "objective/rlhf_reward": 14.828933715820312, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0013184514828026295, + "policy/clipfrac_avg": 0.00809475127607584, + "policy/entropy_avg": 0.17902374267578125, + "step": 701, + "val/clipfrac_avg": 0.18833743035793304, + "val/num_eos_tokens": 358, + "val/ratio": 0.9879262447357178, + "val/ratio_var": 6.120221951277927e-05 + }, + { + "episode": 11232, + "epoch": 0.5672727272727273, + "eps": 0, + "loss/policy_avg": 0.2261451780796051, + "loss/value_avg": 1.5403046607971191, + "lr": 4.320892800523827e-06, + "objective/entropy": -90.61648559570312, + "objective/kl": 27.537485122680664, + "objective/non_score_reward": -1.3768744468688965, + "objective/rlhf_reward": 12.826250076293945, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009536159923300147, + "policy/clipfrac_avg": 0.01545755285769701, + "policy/entropy_avg": 0.20562219619750977, + "step": 702, + "val/clipfrac_avg": 0.033450622111558914, + "val/num_eos_tokens": 595, + "val/ratio": 0.9873440265655518, + "val/ratio_var": 7.613677007611841e-05 + }, + { + "episode": 11248, + "epoch": 0.5680808080808081, + "eps": 0, + "loss/policy_avg": -0.05153674632310867, + "loss/value_avg": 0.18954747915267944, + "lr": 4.307661903773129e-06, + "objective/entropy": -56.119659423828125, + "objective/kl": 43.216548919677734, + "objective/non_score_reward": -2.160827398300171, + "objective/rlhf_reward": 15.80792236328125, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010682737920433283, + "policy/clipfrac_avg": 0.005114596337080002, + "policy/entropy_avg": 0.20781707763671875, + "step": 703, + "val/clipfrac_avg": 0.12178023159503937, + "val/num_eos_tokens": 244, + "val/ratio": 0.9860707521438599, + "val/ratio_var": 5.392924140323885e-05 + }, + { + "episode": 11264, + "epoch": 0.5688888888888889, + "eps": 0, + "loss/policy_avg": 0.22105136513710022, + "loss/value_avg": 0.43452760577201843, + "lr": 4.294435947888405e-06, + "objective/entropy": -180.00924682617188, + "objective/kl": 32.27890396118164, + "objective/non_score_reward": -1.6139451265335083, + "objective/rlhf_reward": 12.425117492675781, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007389815291389823, + "policy/clipfrac_avg": 0.00613048393279314, + "policy/entropy_avg": 0.15887287259101868, + "step": 704, + "val/clipfrac_avg": 0.12273523956537247, + "val/num_eos_tokens": 611, + "val/ratio": 0.9900252819061279, + "val/ratio_var": 6.185931124491617e-05 + }, + { + "episode": 11280, + "epoch": 0.5696969696969697, + "eps": 0, + "loss/policy_avg": -0.044020622968673706, + "loss/value_avg": 1.459969401359558, + "lr": 4.281215027256592e-06, + "objective/entropy": -73.2503890991211, + "objective/kl": 29.207317352294922, + "objective/non_score_reward": -1.460365891456604, + "objective/rlhf_reward": 15.766195297241211, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010228066239506006, + "policy/clipfrac_avg": 0.010731351561844349, + "policy/entropy_avg": 0.17015647888183594, + "step": 705, + "val/clipfrac_avg": 0.1418685019016266, + "val/num_eos_tokens": 378, + "val/ratio": 0.9865360856056213, + "val/ratio_var": 0.0001155980207840912 + }, + { + "episode": 11296, + "epoch": 0.5705050505050505, + "eps": 0, + "loss/policy_avg": 0.10734500735998154, + "loss/value_avg": 0.40052735805511475, + "lr": 4.2679992362287e-06, + "objective/entropy": -91.05632019042969, + "objective/kl": 39.329261779785156, + "objective/non_score_reward": -1.966463327407837, + "objective/rlhf_reward": 12.744474411010742, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010600072564557195, + "policy/clipfrac_avg": 0.005786027293652296, + "policy/entropy_avg": 0.23768234252929688, + "step": 706, + "val/clipfrac_avg": 0.032500892877578735, + "val/num_eos_tokens": 203, + "val/ratio": 0.9860942959785461, + "val/ratio_var": 5.682241317117587e-05 + }, + { + "episode": 11312, + "epoch": 0.5713131313131313, + "eps": 0, + "loss/policy_avg": -0.5062837600708008, + "loss/value_avg": 0.7742470502853394, + "lr": 4.254788669119127e-06, + "objective/entropy": -93.2535400390625, + "objective/kl": 34.540130615234375, + "objective/non_score_reward": -1.727006435394287, + "objective/rlhf_reward": 14.788618087768555, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008475214708596468, + "policy/clipfrac_avg": 0.027025574818253517, + "policy/entropy_avg": 0.21597671508789062, + "step": 707, + "val/clipfrac_avg": 0.1775030940771103, + "val/num_eos_tokens": 165, + "val/ratio": 0.9894334077835083, + "val/ratio_var": 9.222733206115663e-05 + }, + { + "episode": 11328, + "epoch": 0.5721212121212121, + "eps": 0, + "loss/policy_avg": -0.1463669091463089, + "loss/value_avg": 0.5632686614990234, + "lr": 4.241583420204998e-06, + "objective/entropy": -110.37503051757812, + "objective/kl": 36.62909698486328, + "objective/non_score_reward": -1.831454873085022, + "objective/rlhf_reward": 12.89510726928711, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009251610608771443, + "policy/clipfrac_avg": 0.00941617414355278, + "policy/entropy_avg": 0.16801166534423828, + "step": 708, + "val/clipfrac_avg": 0.04514690116047859, + "val/num_eos_tokens": 267, + "val/ratio": 0.9897239208221436, + "val/ratio_var": 7.692311191931367e-05 + }, + { + "episode": 11344, + "epoch": 0.572929292929293, + "eps": 0, + "loss/policy_avg": -0.35295569896698, + "loss/value_avg": 0.7035583257675171, + "lr": 4.228383583725471e-06, + "objective/entropy": -76.27204132080078, + "objective/kl": 31.900005340576172, + "objective/non_score_reward": -1.5950002670288086, + "objective/rlhf_reward": 14.053436279296875, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010020419722422957, + "policy/clipfrac_avg": 0.011507621966302395, + "policy/entropy_avg": 0.16324424743652344, + "step": 709, + "val/clipfrac_avg": 0.12508229911327362, + "val/num_eos_tokens": 173, + "val/ratio": 0.9896982908248901, + "val/ratio_var": 4.9723090342013165e-05 + }, + { + "episode": 11360, + "epoch": 0.5737373737373738, + "eps": 0, + "loss/policy_avg": 0.160404771566391, + "loss/value_avg": 1.1914963722229004, + "lr": 4.215189253881093e-06, + "objective/entropy": -79.55085754394531, + "objective/kl": 36.81852722167969, + "objective/non_score_reward": -1.8409266471862793, + "objective/rlhf_reward": 12.159073829650879, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0011080782860517502, + "policy/clipfrac_avg": 0.006503967102617025, + "policy/entropy_avg": 0.19612884521484375, + "step": 710, + "val/clipfrac_avg": 0.08140315115451813, + "val/num_eos_tokens": 538, + "val/ratio": 0.9866147041320801, + "val/ratio_var": 7.869482942624018e-05 + }, + { + "episode": 11376, + "epoch": 0.5745454545454546, + "eps": 0, + "loss/policy_avg": 0.07548795640468597, + "loss/value_avg": 0.5138013362884521, + "lr": 4.2020005248331056e-06, + "objective/entropy": -119.03164672851562, + "objective/kl": 34.31198501586914, + "objective/non_score_reward": -1.7155992984771729, + "objective/rlhf_reward": 13.503150939941406, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007362160831689835, + "policy/clipfrac_avg": 0.00589254405349493, + "policy/entropy_avg": 0.14674091339111328, + "step": 711, + "val/clipfrac_avg": 0.02668866142630577, + "val/num_eos_tokens": 515, + "val/ratio": 0.990146279335022, + "val/ratio_var": 3.9673428545938805e-05 + }, + { + "episode": 11392, + "epoch": 0.5753535353535354, + "eps": 0, + "loss/policy_avg": -0.2925986051559448, + "loss/value_avg": 0.3821415603160858, + "lr": 4.1888174907027775e-06, + "objective/entropy": -112.06816101074219, + "objective/kl": 38.006500244140625, + "objective/non_score_reward": -1.900325059890747, + "objective/rlhf_reward": 13.990300178527832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.000874843040946871, + "policy/clipfrac_avg": 0.007544378750026226, + "policy/entropy_avg": 0.16875195503234863, + "step": 712, + "val/clipfrac_avg": 0.025153353810310364, + "val/num_eos_tokens": 4, + "val/ratio": 0.9893897771835327, + "val/ratio_var": 8.519255788996816e-05 + }, + { + "episode": 11408, + "epoch": 0.5761616161616162, + "eps": 0, + "loss/policy_avg": -0.5444605350494385, + "loss/value_avg": 1.517474889755249, + "lr": 4.175640245570742e-06, + "objective/entropy": -92.28114318847656, + "objective/kl": 35.69511413574219, + "objective/non_score_reward": -1.7847557067871094, + "objective/rlhf_reward": 13.73868179321289, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008441185345873237, + "policy/clipfrac_avg": 0.005670716054737568, + "policy/entropy_avg": 0.1591033935546875, + "step": 713, + "val/clipfrac_avg": 0.1791025698184967, + "val/num_eos_tokens": 98, + "val/ratio": 0.9903852939605713, + "val/ratio_var": 4.043114677187987e-05 + }, + { + "episode": 11424, + "epoch": 0.576969696969697, + "eps": 0, + "loss/policy_avg": -0.3059786260128021, + "loss/value_avg": 0.6795393228530884, + "lr": 4.162468883476319e-06, + "objective/entropy": -193.77468872070312, + "objective/kl": 42.17440414428711, + "objective/non_score_reward": -2.108720302581787, + "objective/rlhf_reward": 12.281905174255371, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008298080647364259, + "policy/clipfrac_avg": 0.00635810662060976, + "policy/entropy_avg": 0.15823698043823242, + "step": 714, + "val/clipfrac_avg": 0.0064142923802137375, + "val/num_eos_tokens": 1666, + "val/ratio": 0.9892975091934204, + "val/ratio_var": 6.228621350601315e-05 + }, + { + "episode": 11440, + "epoch": 0.5777777777777777, + "eps": 0, + "loss/policy_avg": -0.023257479071617126, + "loss/value_avg": 0.18199321627616882, + "lr": 4.149303498416838e-06, + "objective/entropy": -22.63348388671875, + "objective/kl": 44.308345794677734, + "objective/non_score_reward": -2.2154173851013184, + "objective/rlhf_reward": 14.82364559173584, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0012546201469376683, + "policy/clipfrac_avg": 0.007735411170870066, + "policy/entropy_avg": 0.240936279296875, + "step": 715, + "val/clipfrac_avg": 0.0028473599813878536, + "val/num_eos_tokens": 302, + "val/ratio": 0.9840776920318604, + "val/ratio_var": 5.949061960563995e-05 + }, + { + "episode": 11456, + "epoch": 0.5785858585858585, + "eps": 0, + "loss/policy_avg": 0.3237575888633728, + "loss/value_avg": 0.38197559118270874, + "lr": 4.136144184346979e-06, + "objective/entropy": -168.4746856689453, + "objective/kl": 40.63113784790039, + "objective/non_score_reward": -2.031557083129883, + "objective/rlhf_reward": 13.070005416870117, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0007934231543913484, + "policy/clipfrac_avg": 0.012050900608301163, + "policy/entropy_avg": 0.14413070678710938, + "step": 716, + "val/clipfrac_avg": 0.09322839975357056, + "val/num_eos_tokens": 1174, + "val/ratio": 0.9900047779083252, + "val/ratio_var": 5.29003118572291e-05 + }, + { + "episode": 11472, + "epoch": 0.5793939393939394, + "eps": 0, + "loss/policy_avg": 0.23654325306415558, + "loss/value_avg": 1.166879653930664, + "lr": 4.122991035178093e-06, + "objective/entropy": -96.77055358886719, + "objective/kl": 42.17526626586914, + "objective/non_score_reward": -2.1087632179260254, + "objective/rlhf_reward": 14.500611305236816, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000815618084743619, + "policy/clipfrac_avg": 0.008080050349235535, + "policy/entropy_avg": 0.14806270599365234, + "step": 717, + "val/clipfrac_avg": 0.02973807230591774, + "val/num_eos_tokens": 297, + "val/ratio": 0.9899397492408752, + "val/ratio_var": 3.871042645187117e-05 + }, + { + "episode": 11488, + "epoch": 0.5802020202020202, + "eps": 0, + "loss/policy_avg": -0.11968575417995453, + "loss/value_avg": 0.9426589608192444, + "lr": 4.109844144777537e-06, + "objective/entropy": -58.205047607421875, + "objective/kl": 42.6419792175293, + "objective/non_score_reward": -2.132099151611328, + "objective/rlhf_reward": 12.203838348388672, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009566791704855859, + "policy/clipfrac_avg": 0.0025268401950597763, + "policy/entropy_avg": 0.18893003463745117, + "step": 718, + "val/clipfrac_avg": 0.1098824143409729, + "val/num_eos_tokens": 16, + "val/ratio": 0.9869214296340942, + "val/ratio_var": 8.384165266761556e-05 + }, + { + "episode": 11504, + "epoch": 0.581010101010101, + "eps": 0, + "loss/policy_avg": 0.0208362378180027, + "loss/value_avg": 0.8499202728271484, + "lr": 4.096703606968007e-06, + "objective/entropy": -105.71292114257812, + "objective/kl": 40.70527648925781, + "objective/non_score_reward": -2.035263776779175, + "objective/rlhf_reward": 13.605360984802246, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008132618968375027, + "policy/clipfrac_avg": 0.004117591306567192, + "policy/entropy_avg": 0.15980243682861328, + "step": 719, + "val/clipfrac_avg": 0.04368842393159866, + "val/num_eos_tokens": 428, + "val/ratio": 0.9895676374435425, + "val/ratio_var": 6.342094275169075e-05 + }, + { + "episode": 11520, + "epoch": 0.5818181818181818, + "eps": 0, + "loss/policy_avg": -0.11627452820539474, + "loss/value_avg": 0.7368168830871582, + "lr": 4.083569515526853e-06, + "objective/entropy": -100.23733520507812, + "objective/kl": 32.93721008300781, + "objective/non_score_reward": -1.6468604803085327, + "objective/rlhf_reward": 15.134389877319336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008385346736758947, + "policy/clipfrac_avg": 0.009182492271065712, + "policy/entropy_avg": 0.17890548706054688, + "step": 720, + "val/clipfrac_avg": 0.06867726147174835, + "val/num_eos_tokens": 144, + "val/ratio": 0.9896779656410217, + "val/ratio_var": 7.192800694610924e-05 + }, + { + "episode": 11536, + "epoch": 0.5826262626262626, + "eps": 0, + "loss/policy_avg": -0.06581735610961914, + "loss/value_avg": 0.7933100461959839, + "lr": 4.070441964185428e-06, + "objective/entropy": -103.3985366821289, + "objective/kl": 39.985321044921875, + "objective/non_score_reward": -1.9992659091949463, + "objective/rlhf_reward": 14.547609329223633, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000881620217114687, + "policy/clipfrac_avg": 0.004334163852035999, + "policy/entropy_avg": 0.1855926513671875, + "step": 721, + "val/clipfrac_avg": 0.04114583134651184, + "val/num_eos_tokens": 161, + "val/ratio": 0.989449679851532, + "val/ratio_var": 6.215622852323577e-05 + }, + { + "episode": 11552, + "epoch": 0.5834343434343434, + "eps": 0, + "loss/policy_avg": 0.44041353464126587, + "loss/value_avg": 0.7198789119720459, + "lr": 4.05732104662841e-06, + "objective/entropy": -119.77040100097656, + "objective/kl": 36.42585754394531, + "objective/non_score_reward": -1.8212929964065552, + "objective/rlhf_reward": 13.467769622802734, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008070225012488663, + "policy/clipfrac_avg": 0.007022568956017494, + "policy/entropy_avg": 0.15318524837493896, + "step": 722, + "val/clipfrac_avg": 0.05163312703371048, + "val/num_eos_tokens": 304, + "val/ratio": 0.9899815320968628, + "val/ratio_var": 4.889031333732419e-05 + }, + { + "episode": 11568, + "epoch": 0.5842424242424242, + "eps": 0, + "loss/policy_avg": 0.4866091012954712, + "loss/value_avg": 1.1953067779541016, + "lr": 4.04420685649314e-06, + "objective/entropy": -50.46599578857422, + "objective/kl": 39.113037109375, + "objective/non_score_reward": -1.955651879310608, + "objective/rlhf_reward": 13.911535263061523, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010366745991632342, + "policy/clipfrac_avg": 0.006084335036575794, + "policy/entropy_avg": 0.20149445533752441, + "step": 723, + "val/clipfrac_avg": 0.09489050507545471, + "val/num_eos_tokens": 36, + "val/ratio": 0.9867548942565918, + "val/ratio_var": 6.938407750567421e-05 + }, + { + "episode": 11584, + "epoch": 0.585050505050505, + "eps": 0, + "loss/policy_avg": 0.1589544266462326, + "loss/value_avg": 0.568170964717865, + "lr": 4.031099487368937e-06, + "objective/entropy": -112.03081512451172, + "objective/kl": 37.355255126953125, + "objective/non_score_reward": -1.8677629232406616, + "objective/rlhf_reward": 13.49161148071289, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001035548048093915, + "policy/clipfrac_avg": 0.006865138188004494, + "policy/entropy_avg": 0.20564031600952148, + "step": 724, + "val/clipfrac_avg": 0.05233292281627655, + "val/num_eos_tokens": 347, + "val/ratio": 0.9859846830368042, + "val/ratio_var": 0.00010150864545721561 + }, + { + "episode": 11600, + "epoch": 0.5858585858585859, + "eps": 0, + "loss/policy_avg": -0.19635428488254547, + "loss/value_avg": 0.33023789525032043, + "lr": 4.0179990327964526e-06, + "objective/entropy": -69.79931640625, + "objective/kl": 41.90481948852539, + "objective/non_score_reward": -2.095241069793701, + "objective/rlhf_reward": 14.85007095336914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0010360007872805, + "policy/clipfrac_avg": 0.004560987465083599, + "policy/entropy_avg": 0.20290184020996094, + "step": 725, + "val/clipfrac_avg": 0.08375490456819534, + "val/num_eos_tokens": 261, + "val/ratio": 0.98649001121521, + "val/ratio_var": 6.110723916208372e-05 + }, + { + "episode": 11616, + "epoch": 0.5866666666666667, + "eps": 0, + "loss/policy_avg": 0.0999169647693634, + "loss/value_avg": 1.0625457763671875, + "lr": 4.004905586266988e-06, + "objective/entropy": -104.75285339355469, + "objective/kl": 32.01334762573242, + "objective/non_score_reward": -1.6006674766540527, + "objective/rlhf_reward": 12.766520500183105, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010034965816885233, + "policy/clipfrac_avg": 0.007322699762880802, + "policy/entropy_avg": 0.19988441467285156, + "step": 726, + "val/clipfrac_avg": 0.14562752842903137, + "val/num_eos_tokens": 33, + "val/ratio": 0.9887305498123169, + "val/ratio_var": 7.558859942946583e-05 + }, + { + "episode": 11632, + "epoch": 0.5874747474747475, + "eps": 0, + "loss/policy_avg": 0.004431502893567085, + "loss/value_avg": 0.5651569366455078, + "lr": 3.991819241221836e-06, + "objective/entropy": -53.65510940551758, + "objective/kl": 43.23419952392578, + "objective/non_score_reward": -2.161710262298584, + "objective/rlhf_reward": 12.986727714538574, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001010584644973278, + "policy/clipfrac_avg": 0.006680442951619625, + "policy/entropy_avg": 0.17978286743164062, + "step": 727, + "val/clipfrac_avg": 0.028638603165745735, + "val/num_eos_tokens": 25, + "val/ratio": 0.9880765676498413, + "val/ratio_var": 3.508182999212295e-05 + }, + { + "episode": 11648, + "epoch": 0.5882828282828283, + "eps": 0, + "loss/policy_avg": -0.026804931461811066, + "loss/value_avg": 0.3243325352668762, + "lr": 3.978740091051599e-06, + "objective/entropy": -147.60580444335938, + "objective/kl": 40.000404357910156, + "objective/non_score_reward": -2.0000202655792236, + "objective/rlhf_reward": 13.304666519165039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008220034651458263, + "policy/clipfrac_avg": 0.010559421963989735, + "policy/entropy_avg": 0.15192937850952148, + "step": 728, + "val/clipfrac_avg": 0.047058261930942535, + "val/num_eos_tokens": 93, + "val/ratio": 0.9890267252922058, + "val/ratio_var": 7.159873348427936e-05 + }, + { + "episode": 11664, + "epoch": 0.5890909090909091, + "eps": 0, + "loss/policy_avg": 0.061702877283096313, + "loss/value_avg": 0.25130608677864075, + "lr": 3.965668229095546e-06, + "objective/entropy": -75.26506042480469, + "objective/kl": 34.32489013671875, + "objective/non_score_reward": -1.7162445783615112, + "objective/rlhf_reward": 13.9556303024292, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008753975853323936, + "policy/clipfrac_avg": 0.006253514904528856, + "policy/entropy_avg": 0.17108154296875, + "step": 729, + "val/clipfrac_avg": 0.03564208000898361, + "val/num_eos_tokens": 194, + "val/ratio": 0.9883392453193665, + "val/ratio_var": 5.439678716356866e-05 + }, + { + "episode": 11680, + "epoch": 0.5898989898989899, + "eps": 0, + "loss/policy_avg": 0.5118429064750671, + "loss/value_avg": 0.3809865713119507, + "lr": 3.9526037486409275e-06, + "objective/entropy": -211.5462646484375, + "objective/kl": 38.28752136230469, + "objective/non_score_reward": -1.9143760204315186, + "objective/rlhf_reward": 13.062187194824219, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008200224256142974, + "policy/clipfrac_avg": 0.009272422641515732, + "policy/entropy_avg": 0.15618228912353516, + "step": 730, + "val/clipfrac_avg": 0.13058164715766907, + "val/num_eos_tokens": 191, + "val/ratio": 0.9892928600311279, + "val/ratio_var": 5.910802065045573e-05 + }, + { + "episode": 11696, + "epoch": 0.5907070707070707, + "eps": 0, + "loss/policy_avg": -0.575583279132843, + "loss/value_avg": 0.6588011384010315, + "lr": 3.939546742922318e-06, + "objective/entropy": -155.975341796875, + "objective/kl": 33.536251068115234, + "objective/non_score_reward": -1.6768126487731934, + "objective/rlhf_reward": 14.198186874389648, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006373925716616213, + "policy/clipfrac_avg": 0.011547036468982697, + "policy/entropy_avg": 0.15375041961669922, + "step": 731, + "val/clipfrac_avg": 0.1309666782617569, + "val/num_eos_tokens": 393, + "val/ratio": 0.9912899732589722, + "val/ratio_var": 7.494341116398573e-05 + }, + { + "episode": 11712, + "epoch": 0.5915151515151515, + "eps": 0, + "loss/policy_avg": 0.08975643664598465, + "loss/value_avg": 0.4319133758544922, + "lr": 3.926497305120943e-06, + "objective/entropy": -46.366912841796875, + "objective/kl": 38.771400451660156, + "objective/non_score_reward": -1.9385700225830078, + "objective/rlhf_reward": 14.631742477416992, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010276802349835634, + "policy/clipfrac_avg": 0.004419120494276285, + "policy/entropy_avg": 0.19952774047851562, + "step": 732, + "val/clipfrac_avg": 0.02601846680045128, + "val/num_eos_tokens": 121, + "val/ratio": 0.9856929779052734, + "val/ratio_var": 5.2173218136886135e-05 + }, + { + "episode": 11728, + "epoch": 0.5923232323232324, + "eps": 0, + "loss/policy_avg": -0.5671601295471191, + "loss/value_avg": 1.4009994268417358, + "lr": 3.913455528364027e-06, + "objective/entropy": -30.65240478515625, + "objective/kl": 34.644386291503906, + "objective/non_score_reward": -1.7322194576263428, + "objective/rlhf_reward": 16.127155303955078, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009229708230122924, + "policy/clipfrac_avg": 0.0053136469796299934, + "policy/entropy_avg": 0.1684565544128418, + "step": 733, + "val/clipfrac_avg": 0.16779303550720215, + "val/num_eos_tokens": 257, + "val/ratio": 0.98807692527771, + "val/ratio_var": 4.935808829031885e-05 + }, + { + "episode": 11744, + "epoch": 0.5931313131313132, + "eps": 0, + "loss/policy_avg": 0.1117314025759697, + "loss/value_avg": 0.6027213335037231, + "lr": 3.900421505724119e-06, + "objective/entropy": -115.830078125, + "objective/kl": 38.92997741699219, + "objective/non_score_reward": -1.9464988708496094, + "objective/rlhf_reward": 12.38162612915039, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006995117873884737, + "policy/clipfrac_avg": 0.0067988173104822636, + "policy/entropy_avg": 0.12786293029785156, + "step": 734, + "val/clipfrac_avg": 0.11902365833520889, + "val/num_eos_tokens": 256, + "val/ratio": 0.9913206696510315, + "val/ratio_var": 3.536290387273766e-05 + }, + { + "episode": 11760, + "epoch": 0.593939393939394, + "eps": 0, + "loss/policy_avg": 0.07707646489143372, + "loss/value_avg": 0.8557188510894775, + "lr": 3.887395330218429e-06, + "objective/entropy": -77.05795288085938, + "objective/kl": 34.79220199584961, + "objective/non_score_reward": -1.739610195159912, + "objective/rlhf_reward": 12.510390281677246, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009019911522045732, + "policy/clipfrac_avg": 0.00438590906560421, + "policy/entropy_avg": 0.18432998657226562, + "step": 735, + "val/clipfrac_avg": 0.20910227298736572, + "val/num_eos_tokens": 339, + "val/ratio": 0.9879709482192993, + "val/ratio_var": 7.31311592971906e-05 + }, + { + "episode": 11776, + "epoch": 0.5947474747474748, + "eps": 0, + "loss/policy_avg": 0.005743637681007385, + "loss/value_avg": 0.4823000729084015, + "lr": 3.8743770948081675e-06, + "objective/entropy": -189.58908081054688, + "objective/kl": 40.491024017333984, + "objective/non_score_reward": -2.0245513916015625, + "objective/rlhf_reward": 12.608261108398438, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010324950562790036, + "policy/clipfrac_avg": 0.008821255527436733, + "policy/entropy_avg": 0.1864786148071289, + "step": 736, + "val/clipfrac_avg": 0.03615584969520569, + "val/num_eos_tokens": 146, + "val/ratio": 0.9867680072784424, + "val/ratio_var": 0.00012203787628095597 + }, + { + "episode": 11792, + "epoch": 0.5955555555555555, + "eps": 0, + "loss/policy_avg": 0.11549818515777588, + "loss/value_avg": 0.4191131591796875, + "lr": 3.861366892397878e-06, + "objective/entropy": -53.57340621948242, + "objective/kl": 47.2681999206543, + "objective/non_score_reward": -2.363409996032715, + "objective/rlhf_reward": 11.238153457641602, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0010416920995339751, + "policy/clipfrac_avg": 0.006328020244836807, + "policy/entropy_avg": 0.21573257446289062, + "step": 737, + "val/clipfrac_avg": 0.05454040318727493, + "val/num_eos_tokens": 145, + "val/ratio": 0.9863517880439758, + "val/ratio_var": 5.17384396516718e-05 + }, + { + "episode": 11808, + "epoch": 0.5963636363636363, + "eps": 0, + "loss/policy_avg": 0.09252296388149261, + "loss/value_avg": 0.24368464946746826, + "lr": 3.848364815834782e-06, + "objective/entropy": -9.551445007324219, + "objective/kl": 41.81903839111328, + "objective/non_score_reward": -2.090951919555664, + "objective/rlhf_reward": 15.362173080444336, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.001077502267435193, + "policy/clipfrac_avg": 0.007244379259645939, + "policy/entropy_avg": 0.208465576171875, + "step": 738, + "val/clipfrac_avg": 0.046727634966373444, + "val/num_eos_tokens": 70, + "val/ratio": 0.9859708547592163, + "val/ratio_var": 4.752089807880111e-05 + }, + { + "episode": 11824, + "epoch": 0.5971717171717171, + "eps": 0, + "loss/policy_avg": -0.47144564986228943, + "loss/value_avg": 0.9659825563430786, + "lr": 3.835370957908108e-06, + "objective/entropy": -109.4366226196289, + "objective/kl": 32.577239990234375, + "objective/non_score_reward": -1.6288621425628662, + "objective/rlhf_reward": 13.332075119018555, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009311667527072132, + "policy/clipfrac_avg": 0.007746369112282991, + "policy/entropy_avg": 0.2024822235107422, + "step": 739, + "val/clipfrac_avg": 0.1542586386203766, + "val/num_eos_tokens": 140, + "val/ratio": 0.9888983964920044, + "val/ratio_var": 8.097994577838108e-05 + }, + { + "episode": 11840, + "epoch": 0.597979797979798, + "eps": 0, + "loss/policy_avg": 0.3934873938560486, + "loss/value_avg": 0.5160516500473022, + "lr": 3.822385411348429e-06, + "objective/entropy": -82.36312103271484, + "objective/kl": 38.616180419921875, + "objective/non_score_reward": -1.9308089017868042, + "objective/rlhf_reward": 12.725440979003906, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008991706417873502, + "policy/clipfrac_avg": 0.01843995228409767, + "policy/entropy_avg": 0.2245502471923828, + "step": 740, + "val/clipfrac_avg": 0.0067659346386790276, + "val/num_eos_tokens": 30, + "val/ratio": 0.988348662853241, + "val/ratio_var": 5.281575795379467e-05 + }, + { + "episode": 11856, + "epoch": 0.5987878787878788, + "eps": 0, + "loss/policy_avg": -0.06747608631849289, + "loss/value_avg": 0.3114325702190399, + "lr": 3.809408268827009e-06, + "objective/entropy": -89.22055053710938, + "objective/kl": 46.57951736450195, + "objective/non_score_reward": -2.3289759159088135, + "objective/rlhf_reward": 13.483524322509766, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000996086047962308, + "policy/clipfrac_avg": 0.005896539893001318, + "policy/entropy_avg": 0.19659423828125, + "step": 741, + "val/clipfrac_avg": 0.0699368417263031, + "val/num_eos_tokens": 107, + "val/ratio": 0.9875314235687256, + "val/ratio_var": 4.793483094545081e-05 + }, + { + "episode": 11872, + "epoch": 0.5995959595959596, + "eps": 0, + "loss/policy_avg": -0.2737995684146881, + "loss/value_avg": 0.4772384762763977, + "lr": 3.7964396229551365e-06, + "objective/entropy": -104.62603759765625, + "objective/kl": 36.626800537109375, + "objective/non_score_reward": -1.831339955329895, + "objective/rlhf_reward": 14.778034210205078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008760478813201189, + "policy/clipfrac_avg": 0.005249073728919029, + "policy/entropy_avg": 0.17531394958496094, + "step": 742, + "val/clipfrac_avg": 0.06328597664833069, + "val/num_eos_tokens": 91, + "val/ratio": 0.9885821342468262, + "val/ratio_var": 5.911157131777145e-05 + }, + { + "episode": 11888, + "epoch": 0.6004040404040404, + "eps": 0, + "loss/policy_avg": -0.11605305969715118, + "loss/value_avg": 0.30427810549736023, + "lr": 3.783479566283457e-06, + "objective/entropy": -123.75762176513672, + "objective/kl": 35.787086486816406, + "objective/non_score_reward": -1.7893544435501099, + "objective/rlhf_reward": 14.86689567565918, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007770034717395902, + "policy/clipfrac_avg": 0.0056176697835326195, + "policy/entropy_avg": 0.152313232421875, + "step": 743, + "val/clipfrac_avg": 0.048395052552223206, + "val/num_eos_tokens": 386, + "val/ratio": 0.9905653595924377, + "val/ratio_var": 5.709798642783426e-05 + }, + { + "episode": 11904, + "epoch": 0.6012121212121212, + "eps": 0, + "loss/policy_avg": -0.02294028550386429, + "loss/value_avg": 0.7253668904304504, + "lr": 3.7705281913013286e-06, + "objective/entropy": -39.3416633605957, + "objective/kl": 43.74962615966797, + "objective/non_score_reward": -2.18748140335083, + "objective/rlhf_reward": 12.648456573486328, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010895703453570604, + "policy/clipfrac_avg": 0.0052008070051670074, + "policy/entropy_avg": 0.22774124145507812, + "step": 744, + "val/clipfrac_avg": 0.01829145848751068, + "val/num_eos_tokens": 71, + "val/ratio": 0.985338568687439, + "val/ratio_var": 6.19284255662933e-05 + }, + { + "episode": 11920, + "epoch": 0.602020202020202, + "eps": 0, + "loss/policy_avg": 0.1328388750553131, + "loss/value_avg": 1.3390746116638184, + "lr": 3.7575855904361447e-06, + "objective/entropy": -79.38550567626953, + "objective/kl": 33.780616760253906, + "objective/non_score_reward": -1.6890308856964111, + "objective/rlhf_reward": 14.178156852722168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008661438478156924, + "policy/clipfrac_avg": 0.003558866214007139, + "policy/entropy_avg": 0.15964603424072266, + "step": 745, + "val/clipfrac_avg": 0.05471283197402954, + "val/num_eos_tokens": 81, + "val/ratio": 0.9897934198379517, + "val/ratio_var": 8.215933485189453e-05 + }, + { + "episode": 11936, + "epoch": 0.6028282828282828, + "eps": 0, + "loss/policy_avg": -0.29737263917922974, + "loss/value_avg": 0.6314270496368408, + "lr": 3.7446518560526903e-06, + "objective/entropy": -124.25365447998047, + "objective/kl": 42.908660888671875, + "objective/non_score_reward": -2.145432949066162, + "objective/rlhf_reward": 11.87019157409668, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009470224613323808, + "policy/clipfrac_avg": 0.006878838874399662, + "policy/entropy_avg": 0.1818103790283203, + "step": 746, + "val/clipfrac_avg": 0.0798804834485054, + "val/num_eos_tokens": 329, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 8.261724724434316e-05 + }, + { + "episode": 11952, + "epoch": 0.6036363636363636, + "eps": 0, + "loss/policy_avg": 0.12454540282487869, + "loss/value_avg": 0.4487236738204956, + "lr": 3.731727080452464e-06, + "objective/entropy": -72.31340026855469, + "objective/kl": 43.510276794433594, + "objective/non_score_reward": -2.175513982772827, + "objective/rlhf_reward": 12.347923278808594, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009941391181200743, + "policy/clipfrac_avg": 0.00658874586224556, + "policy/entropy_avg": 0.16569900512695312, + "step": 747, + "val/clipfrac_avg": 0.1540203094482422, + "val/num_eos_tokens": 77, + "val/ratio": 0.9888790845870972, + "val/ratio_var": 2.4041321012191474e-05 + }, + { + "episode": 11968, + "epoch": 0.6044444444444445, + "eps": 0, + "loss/policy_avg": -0.1547861397266388, + "loss/value_avg": 0.49022629857063293, + "lr": 3.7188113558730403e-06, + "objective/entropy": -103.35200500488281, + "objective/kl": 35.28123474121094, + "objective/non_score_reward": -1.764061689376831, + "objective/rlhf_reward": 13.27500057220459, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008191847009584308, + "policy/clipfrac_avg": 0.006343110464513302, + "policy/entropy_avg": 0.14904117584228516, + "step": 748, + "val/clipfrac_avg": 0.04430587589740753, + "val/num_eos_tokens": 124, + "val/ratio": 0.9908789396286011, + "val/ratio_var": 4.545380579656921e-05 + }, + { + "episode": 11984, + "epoch": 0.6052525252525253, + "eps": 0, + "loss/policy_avg": 0.11601711809635162, + "loss/value_avg": 0.6175556182861328, + "lr": 3.705904774487396e-06, + "objective/entropy": -111.66493225097656, + "objective/kl": 36.76349639892578, + "objective/non_score_reward": -1.838174819946289, + "objective/rlhf_reward": 13.693075180053711, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008918631356209517, + "policy/clipfrac_avg": 0.010910662822425365, + "policy/entropy_avg": 0.16454505920410156, + "step": 749, + "val/clipfrac_avg": 0.06755826622247696, + "val/num_eos_tokens": 358, + "val/ratio": 0.9886500835418701, + "val/ratio_var": 6.255816697375849e-05 + }, + { + "episode": 12000, + "epoch": 0.6060606060606061, + "eps": 0, + "loss/policy_avg": -0.04627024754881859, + "loss/value_avg": 0.6224689483642578, + "lr": 3.6930074284032613e-06, + "objective/entropy": -63.2913703918457, + "objective/kl": 32.27269744873047, + "objective/non_score_reward": -1.6136349439620972, + "objective/rlhf_reward": 14.644177436828613, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009490256779827178, + "policy/clipfrac_avg": 0.0068245502188801765, + "policy/entropy_avg": 0.17229938507080078, + "step": 750, + "val/clipfrac_avg": 0.18630267679691315, + "val/num_eos_tokens": 11, + "val/ratio": 0.9878020882606506, + "val/ratio_var": 5.455148493638262e-05 + }, + { + "episode": 12016, + "epoch": 0.6068686868686869, + "eps": 0, + "loss/policy_avg": -0.17890788614749908, + "loss/value_avg": 0.7140947580337524, + "lr": 3.6801194096624515e-06, + "objective/entropy": -54.77946472167969, + "objective/kl": 34.44614028930664, + "objective/non_score_reward": -1.7223069667816162, + "objective/rlhf_reward": 14.894880294799805, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011779924388974905, + "policy/clipfrac_avg": 0.024402525275945663, + "policy/entropy_avg": 0.2098255157470703, + "step": 751, + "val/clipfrac_avg": 0.04182715713977814, + "val/num_eos_tokens": 42, + "val/ratio": 0.9854939579963684, + "val/ratio_var": 0.00014133901277091354 + }, + { + "episode": 12032, + "epoch": 0.6076767676767677, + "eps": 0, + "loss/policy_avg": -0.0015007667243480682, + "loss/value_avg": 0.2544102668762207, + "lr": 3.6672408102402247e-06, + "objective/entropy": -70.776611328125, + "objective/kl": 37.67994689941406, + "objective/non_score_reward": -1.8839974403381348, + "objective/rlhf_reward": 12.108190536499023, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009373503271490335, + "policy/clipfrac_avg": 0.005210112780332565, + "policy/entropy_avg": 0.18710041046142578, + "step": 752, + "val/clipfrac_avg": 0.021128622815012932, + "val/num_eos_tokens": 218, + "val/ratio": 0.9874061942100525, + "val/ratio_var": 5.212551332078874e-05 + }, + { + "episode": 12048, + "epoch": 0.6084848484848485, + "eps": 0, + "loss/policy_avg": 0.05351484939455986, + "loss/value_avg": 0.28849858045578003, + "lr": 3.654371722044616e-06, + "objective/entropy": -89.17417907714844, + "objective/kl": 44.5589714050293, + "objective/non_score_reward": -2.2279486656188965, + "objective/rlhf_reward": 12.178300857543945, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009375615045428276, + "policy/clipfrac_avg": 0.005882620811462402, + "policy/entropy_avg": 0.18878555297851562, + "step": 753, + "val/clipfrac_avg": 0.021566759794950485, + "val/num_eos_tokens": 120, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 5.727734605898149e-05 + }, + { + "episode": 12064, + "epoch": 0.6092929292929293, + "eps": 0, + "loss/policy_avg": -0.12424173951148987, + "loss/value_avg": 0.359388530254364, + "lr": 3.641512236915783e-06, + "objective/entropy": -114.3979263305664, + "objective/kl": 43.28697967529297, + "objective/non_score_reward": -2.16434907913208, + "objective/rlhf_reward": 14.226276397705078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000866133370436728, + "policy/clipfrac_avg": 0.0050315093249082565, + "policy/entropy_avg": 0.16837501525878906, + "step": 754, + "val/clipfrac_avg": 0.1426960825920105, + "val/num_eos_tokens": 391, + "val/ratio": 0.9894628524780273, + "val/ratio_var": 5.016153590986505e-05 + }, + { + "episode": 12080, + "epoch": 0.6101010101010101, + "eps": 0, + "loss/policy_avg": -0.2821425795555115, + "loss/value_avg": 0.38819342851638794, + "lr": 3.6286624466253496e-06, + "objective/entropy": -164.57290649414062, + "objective/kl": 38.83340072631836, + "objective/non_score_reward": -1.9416700601577759, + "objective/rlhf_reward": 14.245829582214355, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008140793070197105, + "policy/clipfrac_avg": 0.00435154652222991, + "policy/entropy_avg": 0.15729975700378418, + "step": 755, + "val/clipfrac_avg": 0.09525477886199951, + "val/num_eos_tokens": 336, + "val/ratio": 0.989446222782135, + "val/ratio_var": 5.3185020078672096e-05 + }, + { + "episode": 12096, + "epoch": 0.610909090909091, + "eps": 0, + "loss/policy_avg": 0.05094696208834648, + "loss/value_avg": 0.4260476529598236, + "lr": 3.6158224428757538e-06, + "objective/entropy": -61.176116943359375, + "objective/kl": 32.80990219116211, + "objective/non_score_reward": -1.6404950618743896, + "objective/rlhf_reward": 16.24231719970703, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009597806492820382, + "policy/clipfrac_avg": 0.0061972904950380325, + "policy/entropy_avg": 0.16566753387451172, + "step": 756, + "val/clipfrac_avg": 0.01815173029899597, + "val/num_eos_tokens": 163, + "val/ratio": 0.9894044399261475, + "val/ratio_var": 4.711053770733997e-05 + }, + { + "episode": 12112, + "epoch": 0.6117171717171718, + "eps": 0, + "loss/policy_avg": -0.040469832718372345, + "loss/value_avg": 0.3746616244316101, + "lr": 3.602992317299594e-06, + "objective/entropy": -52.30070877075195, + "objective/kl": 34.218990325927734, + "objective/non_score_reward": -1.7109496593475342, + "objective/rlhf_reward": 13.273425102233887, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009450222714804113, + "policy/clipfrac_avg": 0.0049097794108092785, + "policy/entropy_avg": 0.1805877685546875, + "step": 757, + "val/clipfrac_avg": 0.09231511503458023, + "val/num_eos_tokens": 105, + "val/ratio": 0.9877369403839111, + "val/ratio_var": 4.407649248605594e-05 + }, + { + "episode": 12128, + "epoch": 0.6125252525252525, + "eps": 0, + "loss/policy_avg": -0.0790981650352478, + "loss/value_avg": 0.289002388715744, + "lr": 3.5901721614589725e-06, + "objective/entropy": -33.08611297607422, + "objective/kl": 42.89472961425781, + "objective/non_score_reward": -2.1447365283966064, + "objective/rlhf_reward": 14.792763710021973, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011628612410277128, + "policy/clipfrac_avg": 0.00601824838668108, + "policy/entropy_avg": 0.21850204467773438, + "step": 758, + "val/clipfrac_avg": 0.05997534096240997, + "val/num_eos_tokens": 208, + "val/ratio": 0.9858998656272888, + "val/ratio_var": 4.6025765186641365e-05 + }, + { + "episode": 12144, + "epoch": 0.6133333333333333, + "eps": 0, + "loss/policy_avg": 0.08223091810941696, + "loss/value_avg": 0.4521709978580475, + "lr": 3.5773620668448384e-06, + "objective/entropy": -57.002418518066406, + "objective/kl": 34.86107635498047, + "objective/non_score_reward": -1.743053674697876, + "objective/rlhf_reward": 12.421009063720703, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009696290362626314, + "policy/clipfrac_avg": 0.005794525146484375, + "policy/entropy_avg": 0.1826324462890625, + "step": 759, + "val/clipfrac_avg": 0.03934604302048683, + "val/num_eos_tokens": 137, + "val/ratio": 0.9880379438400269, + "val/ratio_var": 4.820780668524094e-05 + }, + { + "episode": 12160, + "epoch": 0.6141414141414141, + "eps": 0, + "loss/policy_avg": 0.012903919443488121, + "loss/value_avg": 0.22128555178642273, + "lr": 3.564562124876344e-06, + "objective/entropy": -58.77183532714844, + "objective/kl": 40.15751647949219, + "objective/non_score_reward": -2.007875919342041, + "objective/rlhf_reward": 13.171812057495117, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009755170904099941, + "policy/clipfrac_avg": 0.0075699929147958755, + "policy/entropy_avg": 0.1793661117553711, + "step": 760, + "val/clipfrac_avg": 0.007695037871599197, + "val/num_eos_tokens": 194, + "val/ratio": 0.9879107475280762, + "val/ratio_var": 9.122795017901808e-05 + }, + { + "episode": 12176, + "epoch": 0.6149494949494949, + "eps": 0, + "loss/policy_avg": 0.23269692063331604, + "loss/value_avg": 0.6034168004989624, + "lr": 3.5517724269001853e-06, + "objective/entropy": -92.11294555664062, + "objective/kl": 36.40907287597656, + "objective/non_score_reward": -1.8204538822174072, + "objective/rlhf_reward": 13.468608856201172, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009949133964255452, + "policy/clipfrac_avg": 0.012031201273202896, + "policy/entropy_avg": 0.19497203826904297, + "step": 761, + "val/clipfrac_avg": 0.03142271935939789, + "val/num_eos_tokens": 210, + "val/ratio": 0.9871051907539368, + "val/ratio_var": 7.474326412193477e-05 + }, + { + "episode": 12192, + "epoch": 0.6157575757575757, + "eps": 0, + "loss/policy_avg": -0.014487015083432198, + "loss/value_avg": 1.0809375047683716, + "lr": 3.538993064189954e-06, + "objective/entropy": -42.83490753173828, + "objective/kl": 32.94031524658203, + "objective/non_score_reward": -1.6470158100128174, + "objective/rlhf_reward": 13.567828178405762, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0011591427028179169, + "policy/clipfrac_avg": 0.004484353121370077, + "policy/entropy_avg": 0.21469688415527344, + "step": 762, + "val/clipfrac_avg": 0.1017560362815857, + "val/num_eos_tokens": 453, + "val/ratio": 0.9864501953125, + "val/ratio_var": 8.947903552325442e-05 + }, + { + "episode": 12208, + "epoch": 0.6165656565656565, + "eps": 0, + "loss/policy_avg": -0.0410991832613945, + "loss/value_avg": 1.3014378547668457, + "lr": 3.526224127945479e-06, + "objective/entropy": -71.51974487304688, + "objective/kl": 31.28769302368164, + "objective/non_score_reward": -1.5643846988677979, + "objective/rlhf_reward": 15.216865539550781, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007811100222170353, + "policy/clipfrac_avg": 0.0053521376103162766, + "policy/entropy_avg": 0.16764509677886963, + "step": 763, + "val/clipfrac_avg": 0.059704042971134186, + "val/num_eos_tokens": 203, + "val/ratio": 0.9894566535949707, + "val/ratio_var": 6.841208232799545e-05 + }, + { + "episode": 12224, + "epoch": 0.6173737373737374, + "eps": 0, + "loss/policy_avg": 0.112126424908638, + "loss/value_avg": 0.725554347038269, + "lr": 3.5134657092921877e-06, + "objective/entropy": -99.9854736328125, + "objective/kl": 41.09782028198242, + "objective/non_score_reward": -2.0548908710479736, + "objective/rlhf_reward": 10.624795913696289, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009035035036504269, + "policy/clipfrac_avg": 0.010427422821521759, + "policy/entropy_avg": 0.17998504638671875, + "step": 764, + "val/clipfrac_avg": 0.12764863669872284, + "val/num_eos_tokens": 226, + "val/ratio": 0.988477349281311, + "val/ratio_var": 8.909796451916918e-05 + }, + { + "episode": 12240, + "epoch": 0.6181818181818182, + "eps": 0, + "loss/policy_avg": 0.37758830189704895, + "loss/value_avg": 0.7465800046920776, + "lr": 3.500717899280442e-06, + "objective/entropy": -135.23390197753906, + "objective/kl": 41.466583251953125, + "objective/non_score_reward": -2.073329210281372, + "objective/rlhf_reward": 12.94229507446289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009621024364605546, + "policy/clipfrac_avg": 0.00628144945949316, + "policy/entropy_avg": 0.17942047119140625, + "step": 765, + "val/clipfrac_avg": 0.10096652805805206, + "val/num_eos_tokens": 233, + "val/ratio": 0.9877568483352661, + "val/ratio_var": 7.067532715154812e-05 + }, + { + "episode": 12256, + "epoch": 0.618989898989899, + "eps": 0, + "loss/policy_avg": -0.10782650113105774, + "loss/value_avg": 0.4996359944343567, + "lr": 3.4879807888849026e-06, + "objective/entropy": -86.19502258300781, + "objective/kl": 39.61420440673828, + "objective/non_score_reward": -1.9807100296020508, + "objective/rlhf_reward": 13.706789016723633, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009516358841210604, + "policy/clipfrac_avg": 0.01008635200560093, + "policy/entropy_avg": 0.17633438110351562, + "step": 766, + "val/clipfrac_avg": 0.05575938522815704, + "val/num_eos_tokens": 426, + "val/ratio": 0.9884860515594482, + "val/ratio_var": 5.413329563452862e-05 + }, + { + "episode": 12272, + "epoch": 0.6197979797979798, + "eps": 0, + "loss/policy_avg": 0.0611935630440712, + "loss/value_avg": 0.40072089433670044, + "lr": 3.475254469003865e-06, + "objective/entropy": -78.26728057861328, + "objective/kl": 38.48719787597656, + "objective/non_score_reward": -1.9243600368499756, + "objective/rlhf_reward": 11.661577224731445, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009466722258366644, + "policy/clipfrac_avg": 0.0042708818800747395, + "policy/entropy_avg": 0.17498779296875, + "step": 767, + "val/clipfrac_avg": 0.1151452288031578, + "val/num_eos_tokens": 415, + "val/ratio": 0.9874659776687622, + "val/ratio_var": 7.443072536261752e-05 + }, + { + "episode": 12288, + "epoch": 0.6206060606060606, + "eps": 0, + "loss/policy_avg": -0.07818017899990082, + "loss/value_avg": 0.588293731212616, + "lr": 3.4625390304586224e-06, + "objective/entropy": -45.31693649291992, + "objective/kl": 41.06260681152344, + "objective/non_score_reward": -2.0531303882598877, + "objective/rlhf_reward": 13.860933303833008, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0011651598615571856, + "policy/clipfrac_avg": 0.0047182347625494, + "policy/entropy_avg": 0.2090606689453125, + "step": 768, + "val/clipfrac_avg": 0.16338574886322021, + "val/num_eos_tokens": 9, + "val/ratio": 0.986497700214386, + "val/ratio_var": 8.175932453013957e-05 + }, + { + "episode": 12304, + "epoch": 0.6214141414141414, + "eps": 0, + "loss/policy_avg": -0.3265213966369629, + "loss/value_avg": 0.5878744125366211, + "lr": 3.4498345639928167e-06, + "objective/entropy": -103.96510314941406, + "objective/kl": 35.794551849365234, + "objective/non_score_reward": -1.7897276878356934, + "objective/rlhf_reward": 13.507146835327148, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010084104724228382, + "policy/clipfrac_avg": 0.0070042358711361885, + "policy/entropy_avg": 0.19075870513916016, + "step": 769, + "val/clipfrac_avg": 0.12878292798995972, + "val/num_eos_tokens": 8, + "val/ratio": 0.9861502051353455, + "val/ratio_var": 8.973718649940565e-05 + }, + { + "episode": 12320, + "epoch": 0.6222222222222222, + "eps": 0, + "loss/policy_avg": -0.24864813685417175, + "loss/value_avg": 0.703667402267456, + "lr": 3.4371411602717785e-06, + "objective/entropy": -92.59934997558594, + "objective/kl": 31.87890625, + "objective/non_score_reward": -1.5939452648162842, + "objective/rlhf_reward": 14.429492950439453, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007836055592633784, + "policy/clipfrac_avg": 0.004061062820255756, + "policy/entropy_avg": 0.13613224029541016, + "step": 770, + "val/clipfrac_avg": 0.14356926083564758, + "val/num_eos_tokens": 22, + "val/ratio": 0.9909623265266418, + "val/ratio_var": 6.252773891901597e-05 + }, + { + "episode": 12336, + "epoch": 0.623030303030303, + "eps": 0, + "loss/policy_avg": -0.11106158792972565, + "loss/value_avg": 0.7719787359237671, + "lr": 3.424458909881897e-06, + "objective/entropy": -201.825927734375, + "objective/kl": 32.05082321166992, + "objective/non_score_reward": -1.602541208267212, + "objective/rlhf_reward": 13.842771530151367, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009224842651747167, + "policy/clipfrac_avg": 0.004090993665158749, + "policy/entropy_avg": 0.19452500343322754, + "step": 771, + "val/clipfrac_avg": 0.06381858140230179, + "val/num_eos_tokens": 137, + "val/ratio": 0.9873221516609192, + "val/ratio_var": 0.00010800129530252889 + }, + { + "episode": 12352, + "epoch": 0.6238383838383839, + "eps": 0, + "loss/policy_avg": -0.3147702217102051, + "loss/value_avg": 1.0198739767074585, + "lr": 3.411787903329963e-06, + "objective/entropy": -97.80130767822266, + "objective/kl": 28.94087791442871, + "objective/non_score_reward": -1.4470438957214355, + "objective/rlhf_reward": 13.572486877441406, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008233180269598961, + "policy/clipfrac_avg": 0.007523335050791502, + "policy/entropy_avg": 0.17452716827392578, + "step": 772, + "val/clipfrac_avg": 0.021688329055905342, + "val/num_eos_tokens": 116, + "val/ratio": 0.9885031580924988, + "val/ratio_var": 6.793635111534968e-05 + }, + { + "episode": 12368, + "epoch": 0.6246464646464647, + "eps": 0, + "loss/policy_avg": -0.019919998943805695, + "loss/value_avg": 0.31691867113113403, + "lr": 3.3991282310425276e-06, + "objective/entropy": -61.794166564941406, + "objective/kl": 33.001129150390625, + "objective/non_score_reward": -1.6500564813613892, + "objective/rlhf_reward": 12.943693161010742, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009417567634955049, + "policy/clipfrac_avg": 0.005282199941575527, + "policy/entropy_avg": 0.1732311248779297, + "step": 773, + "val/clipfrac_avg": 0.05137361213564873, + "val/num_eos_tokens": 13, + "val/ratio": 0.9889352321624756, + "val/ratio_var": 3.9570189983351156e-05 + }, + { + "episode": 12384, + "epoch": 0.6254545454545455, + "eps": 0, + "loss/policy_avg": 0.010051839053630829, + "loss/value_avg": 0.5414019227027893, + "lr": 3.386479983365251e-06, + "objective/entropy": -73.91504669189453, + "objective/kl": 29.43255615234375, + "objective/non_score_reward": -1.4716278314590454, + "objective/rlhf_reward": 13.293996810913086, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0010180522222071886, + "policy/clipfrac_avg": 0.0050889430567622185, + "policy/entropy_avg": 0.2153911590576172, + "step": 774, + "val/clipfrac_avg": 0.041913747787475586, + "val/num_eos_tokens": 214, + "val/ratio": 0.986315906047821, + "val/ratio_var": 8.102346328087151e-05 + }, + { + "episode": 12400, + "epoch": 0.6262626262626263, + "eps": 0, + "loss/policy_avg": -0.040474407374858856, + "loss/value_avg": 0.30427631735801697, + "lr": 3.3738432505622653e-06, + "objective/entropy": -75.78406524658203, + "objective/kl": 31.846160888671875, + "objective/non_score_reward": -1.5923081636428833, + "objective/rlhf_reward": 11.946754455566406, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009044273756444454, + "policy/clipfrac_avg": 0.005002772435545921, + "policy/entropy_avg": 0.18775272369384766, + "step": 775, + "val/clipfrac_avg": 0.04153668135404587, + "val/num_eos_tokens": 243, + "val/ratio": 0.9877529144287109, + "val/ratio_var": 5.462392800836824e-05 + }, + { + "episode": 12416, + "epoch": 0.6270707070707071, + "eps": 0, + "loss/policy_avg": -0.25257226824760437, + "loss/value_avg": 0.47636741399765015, + "lr": 3.3612181228155237e-06, + "objective/entropy": -97.84140014648438, + "objective/kl": 35.974266052246094, + "objective/non_score_reward": -1.798713207244873, + "objective/rlhf_reward": 13.685661315917969, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009363207500427961, + "policy/clipfrac_avg": 0.0035052683670073748, + "policy/entropy_avg": 0.18935298919677734, + "step": 776, + "val/clipfrac_avg": 0.07260937243700027, + "val/num_eos_tokens": 26, + "val/ratio": 0.9877586364746094, + "val/ratio_var": 0.00010292992374161258 + }, + { + "episode": 12432, + "epoch": 0.6278787878787879, + "eps": 0, + "loss/policy_avg": 0.41764986515045166, + "loss/value_avg": 0.8629061579704285, + "lr": 3.3486046902241663e-06, + "objective/entropy": -40.24072265625, + "objective/kl": 29.197486877441406, + "objective/non_score_reward": -1.4598743915557861, + "objective/rlhf_reward": 12.883874893188477, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010831760009750724, + "policy/clipfrac_avg": 0.02700810506939888, + "policy/entropy_avg": 0.16212844848632812, + "step": 777, + "val/clipfrac_avg": 0.08423963189125061, + "val/num_eos_tokens": 192, + "val/ratio": 0.9885735511779785, + "val/ratio_var": 8.724266081117094e-05 + }, + { + "episode": 12448, + "epoch": 0.6286868686868687, + "eps": 0, + "loss/policy_avg": 0.2242026925086975, + "loss/value_avg": 0.9421164989471436, + "lr": 3.336003042803861e-06, + "objective/entropy": -69.478271484375, + "objective/kl": 33.57006072998047, + "objective/non_score_reward": -1.6785030364990234, + "objective/rlhf_reward": 14.055871963500977, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008899248205125332, + "policy/clipfrac_avg": 0.006825867109000683, + "policy/entropy_avg": 0.14767837524414062, + "step": 778, + "val/clipfrac_avg": 0.1501787304878235, + "val/num_eos_tokens": 109, + "val/ratio": 0.9905418753623962, + "val/ratio_var": 4.142183388466947e-05 + }, + { + "episode": 12464, + "epoch": 0.6294949494949494, + "eps": 0, + "loss/policy_avg": 0.03614666312932968, + "loss/value_avg": 0.7387021780014038, + "lr": 3.3234132704861786e-06, + "objective/entropy": -182.4083251953125, + "objective/kl": 31.944690704345703, + "objective/non_score_reward": -1.5972344875335693, + "objective/rlhf_reward": 12.043390274047852, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.000871892087161541, + "policy/clipfrac_avg": 0.004613575059920549, + "policy/entropy_avg": 0.1733226776123047, + "step": 779, + "val/clipfrac_avg": 0.1070820540189743, + "val/num_eos_tokens": 28, + "val/ratio": 0.9880519509315491, + "val/ratio_var": 9.137512097368017e-05 + }, + { + "episode": 12480, + "epoch": 0.6303030303030303, + "eps": 0, + "loss/policy_avg": 0.491243839263916, + "loss/value_avg": 1.7449349164962769, + "lr": 3.310835463117942e-06, + "objective/entropy": -80.43102264404297, + "objective/kl": 22.97904396057129, + "objective/non_score_reward": -1.1489522457122803, + "objective/rlhf_reward": 13.28073501586914, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000753985601477325, + "policy/clipfrac_avg": 0.013304230757057667, + "policy/entropy_avg": 0.16235482692718506, + "step": 780, + "val/clipfrac_avg": 0.12828537821769714, + "val/num_eos_tokens": 261, + "val/ratio": 0.991666316986084, + "val/ratio_var": 4.57862151961308e-05 + }, + { + "episode": 12496, + "epoch": 0.6311111111111111, + "eps": 0, + "loss/policy_avg": -0.02106083557009697, + "loss/value_avg": 0.26465463638305664, + "lr": 3.2982697104605855e-06, + "objective/entropy": -96.58609008789062, + "objective/kl": 37.96983337402344, + "objective/non_score_reward": -1.898491621017456, + "objective/rlhf_reward": 12.742133140563965, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008919077808968723, + "policy/clipfrac_avg": 0.004138793330639601, + "policy/entropy_avg": 0.176055908203125, + "step": 781, + "val/clipfrac_avg": 0.022776879370212555, + "val/num_eos_tokens": 232, + "val/ratio": 0.9880313873291016, + "val/ratio_var": 5.30094766872935e-05 + }, + { + "episode": 12512, + "epoch": 0.6319191919191919, + "eps": 0, + "loss/policy_avg": -0.21595685184001923, + "loss/value_avg": 0.1668311059474945, + "lr": 3.2857161021895125e-06, + "objective/entropy": -113.97618103027344, + "objective/kl": 30.898021697998047, + "objective/non_score_reward": -1.5449011325836182, + "objective/rlhf_reward": 15.892599105834961, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0008122409926727414, + "policy/clipfrac_avg": 0.007180417887866497, + "policy/entropy_avg": 0.15228557586669922, + "step": 782, + "val/clipfrac_avg": 0.04710119590163231, + "val/num_eos_tokens": 88, + "val/ratio": 0.9905120134353638, + "val/ratio_var": 4.6862616727594286e-05 + }, + { + "episode": 12528, + "epoch": 0.6327272727272727, + "eps": 0, + "loss/policy_avg": -0.02830284647643566, + "loss/value_avg": 0.470698744058609, + "lr": 3.273174727893463e-06, + "objective/entropy": -119.84375762939453, + "objective/kl": 35.46149444580078, + "objective/non_score_reward": -1.773074746131897, + "objective/rlhf_reward": 13.00036334991455, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000846280949190259, + "policy/clipfrac_avg": 0.006307992152869701, + "policy/entropy_avg": 0.15985870361328125, + "step": 783, + "val/clipfrac_avg": 0.09303553402423859, + "val/num_eos_tokens": 242, + "val/ratio": 0.9886760711669922, + "val/ratio_var": 8.341170905623585e-05 + }, + { + "episode": 12544, + "epoch": 0.6335353535353535, + "eps": 0, + "loss/policy_avg": -0.03483296558260918, + "loss/value_avg": 0.2278536707162857, + "lr": 3.2606456770738636e-06, + "objective/entropy": -32.83402633666992, + "objective/kl": 35.053001403808594, + "objective/non_score_reward": -1.7526500225067139, + "objective/rlhf_reward": 13.708287239074707, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010523913661018014, + "policy/clipfrac_avg": 0.005916148889809847, + "policy/entropy_avg": 0.2171039581298828, + "step": 784, + "val/clipfrac_avg": 0.007893375121057034, + "val/num_eos_tokens": 463, + "val/ratio": 0.9859672784805298, + "val/ratio_var": 5.139111090102233e-05 + }, + { + "episode": 12560, + "epoch": 0.6343434343434343, + "eps": 0, + "loss/policy_avg": 0.17171788215637207, + "loss/value_avg": 0.5937256217002869, + "lr": 3.2481290391442034e-06, + "objective/entropy": -109.50154113769531, + "objective/kl": 41.38114929199219, + "objective/non_score_reward": -2.0690574645996094, + "objective/rlhf_reward": 13.82156753540039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009320294484496117, + "policy/clipfrac_avg": 0.005337849725037813, + "policy/entropy_avg": 0.1828467845916748, + "step": 785, + "val/clipfrac_avg": 0.0434221550822258, + "val/num_eos_tokens": 252, + "val/ratio": 0.9880971908569336, + "val/ratio_var": 7.803420157870278e-05 + }, + { + "episode": 12576, + "epoch": 0.6351515151515151, + "eps": 0, + "loss/policy_avg": -0.12326152622699738, + "loss/value_avg": 0.22453081607818604, + "lr": 3.235624903429374e-06, + "objective/entropy": -80.62982940673828, + "objective/kl": 39.33965301513672, + "objective/non_score_reward": -1.9669828414916992, + "objective/rlhf_reward": 14.3767671585083, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009726607240736485, + "policy/clipfrac_avg": 0.005861825309693813, + "policy/entropy_avg": 0.18674850463867188, + "step": 786, + "val/clipfrac_avg": 0.04197417199611664, + "val/num_eos_tokens": 55, + "val/ratio": 0.9867269396781921, + "val/ratio_var": 3.261497477069497e-05 + }, + { + "episode": 12592, + "epoch": 0.635959595959596, + "eps": 0, + "loss/policy_avg": 0.02891068160533905, + "loss/value_avg": 0.2414318025112152, + "lr": 3.2231333591650567e-06, + "objective/entropy": -64.31414031982422, + "objective/kl": 46.58558654785156, + "objective/non_score_reward": -2.3292794227600098, + "objective/rlhf_reward": 14.576970100402832, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009546708315610886, + "policy/clipfrac_avg": 0.00552853848785162, + "policy/entropy_avg": 0.189300537109375, + "step": 787, + "val/clipfrac_avg": 0.0013757861452177167, + "val/num_eos_tokens": 363, + "val/ratio": 0.987205982208252, + "val/ratio_var": 3.5099023079965264e-05 + }, + { + "episode": 12608, + "epoch": 0.6367676767676768, + "eps": 0, + "loss/policy_avg": -0.19538097083568573, + "loss/value_avg": 0.7160595655441284, + "lr": 3.210654495497068e-06, + "objective/entropy": -57.81846237182617, + "objective/kl": 42.85956573486328, + "objective/non_score_reward": -2.1429784297943115, + "objective/rlhf_reward": 12.14999008178711, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0011135776294395328, + "policy/clipfrac_avg": 0.009378955699503422, + "policy/entropy_avg": 0.2034454345703125, + "step": 788, + "val/clipfrac_avg": 0.1690627932548523, + "val/num_eos_tokens": 481, + "val/ratio": 0.9861621856689453, + "val/ratio_var": 9.24188134376891e-05 + }, + { + "episode": 12624, + "epoch": 0.6375757575757576, + "eps": 0, + "loss/policy_avg": 0.048018310219049454, + "loss/value_avg": 0.4312187135219574, + "lr": 3.198188401480734e-06, + "objective/entropy": -76.67124938964844, + "objective/kl": 36.849369049072266, + "objective/non_score_reward": -1.8424686193466187, + "objective/rlhf_reward": 12.25128173828125, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009952555410563946, + "policy/clipfrac_avg": 0.004150367341935635, + "policy/entropy_avg": 0.18071389198303223, + "step": 789, + "val/clipfrac_avg": 0.08804323524236679, + "val/num_eos_tokens": 357, + "val/ratio": 0.9867403507232666, + "val/ratio_var": 0.00013288675108924508 + }, + { + "episode": 12640, + "epoch": 0.6383838383838384, + "eps": 0, + "loss/policy_avg": 0.18971526622772217, + "loss/value_avg": 0.6626036167144775, + "lr": 3.185735166080244e-06, + "objective/entropy": -126.28926086425781, + "objective/kl": 33.96638488769531, + "objective/non_score_reward": -1.6983189582824707, + "objective/rlhf_reward": 12.973556518554688, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0007302669691853225, + "policy/clipfrac_avg": 0.008397979661822319, + "policy/entropy_avg": 0.14786386489868164, + "step": 790, + "val/clipfrac_avg": 0.06916166841983795, + "val/num_eos_tokens": 600, + "val/ratio": 0.9910507798194885, + "val/ratio_var": 5.283964492264204e-05 + }, + { + "episode": 12656, + "epoch": 0.6391919191919192, + "eps": 0, + "loss/policy_avg": -0.4279332160949707, + "loss/value_avg": 0.5117957592010498, + "lr": 3.173294878168025e-06, + "objective/entropy": -75.64164733886719, + "objective/kl": 40.16947555541992, + "objective/non_score_reward": -2.0084738731384277, + "objective/rlhf_reward": 12.86652660369873, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008897786610759795, + "policy/clipfrac_avg": 0.004540015943348408, + "policy/entropy_avg": 0.18616324663162231, + "step": 791, + "val/clipfrac_avg": 0.0338541679084301, + "val/num_eos_tokens": 224, + "val/ratio": 0.9873502254486084, + "val/ratio_var": 6.438903801608831e-05 + }, + { + "episode": 12672, + "epoch": 0.64, + "eps": 0, + "loss/policy_avg": -0.30223172903060913, + "loss/value_avg": 0.7414761185646057, + "lr": 3.160867626524107e-06, + "objective/entropy": -36.1086540222168, + "objective/kl": 35.08636474609375, + "objective/non_score_reward": -1.7543182373046875, + "objective/rlhf_reward": 14.995681762695312, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010164354462176561, + "policy/clipfrac_avg": 0.0033426620066165924, + "policy/entropy_avg": 0.20384597778320312, + "step": 792, + "val/clipfrac_avg": 0.08375494927167892, + "val/num_eos_tokens": 121, + "val/ratio": 0.9862762093544006, + "val/ratio_var": 8.216432615881786e-05 + }, + { + "episode": 12688, + "epoch": 0.6408080808080808, + "eps": 0, + "loss/policy_avg": 0.11556120216846466, + "loss/value_avg": 0.44696569442749023, + "lr": 3.1484534998354832e-06, + "objective/entropy": -52.89218521118164, + "objective/kl": 33.01978302001953, + "objective/non_score_reward": -1.650989294052124, + "objective/rlhf_reward": 12.270885467529297, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009637182811275125, + "policy/clipfrac_avg": 0.006595691200345755, + "policy/entropy_avg": 0.18551349639892578, + "step": 793, + "val/clipfrac_avg": 0.0313958078622818, + "val/num_eos_tokens": 483, + "val/ratio": 0.987555742263794, + "val/ratio_var": 4.9779242544900626e-05 + }, + { + "episode": 12704, + "epoch": 0.6416161616161616, + "eps": 0, + "loss/policy_avg": -0.11985203623771667, + "loss/value_avg": 0.5595318078994751, + "lr": 3.136052586695481e-06, + "objective/entropy": -44.2855110168457, + "objective/kl": 38.74113845825195, + "objective/non_score_reward": -1.9370570182800293, + "objective/rlhf_reward": 13.531692504882812, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009892593370750546, + "policy/clipfrac_avg": 0.006002710200846195, + "policy/entropy_avg": 0.20067405700683594, + "step": 794, + "val/clipfrac_avg": 0.051367055624723434, + "val/num_eos_tokens": 146, + "val/ratio": 0.98707115650177, + "val/ratio_var": 6.210069841472432e-05 + }, + { + "episode": 12720, + "epoch": 0.6424242424242425, + "eps": 0, + "loss/policy_avg": 0.06161116436123848, + "loss/value_avg": 0.20387178659439087, + "lr": 3.12366497560313e-06, + "objective/entropy": -47.063812255859375, + "objective/kl": 36.60810852050781, + "objective/non_score_reward": -1.8304054737091064, + "objective/rlhf_reward": 13.661782264709473, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000998336123302579, + "policy/clipfrac_avg": 0.004183639772236347, + "policy/entropy_avg": 0.19140625, + "step": 795, + "val/clipfrac_avg": 0.08834637701511383, + "val/num_eos_tokens": 342, + "val/ratio": 0.9874628782272339, + "val/ratio_var": 3.3792719477787614e-05 + }, + { + "episode": 12736, + "epoch": 0.6432323232323233, + "eps": 0, + "loss/policy_avg": 0.02652081847190857, + "loss/value_avg": 0.4432865381240845, + "lr": 3.1112907549625304e-06, + "objective/entropy": -104.79203796386719, + "objective/kl": 35.12114715576172, + "objective/non_score_reward": -1.7560575008392334, + "objective/rlhf_reward": 13.486129760742188, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008146626059897244, + "policy/clipfrac_avg": 0.0056906514801084995, + "policy/entropy_avg": 0.1502237319946289, + "step": 796, + "val/clipfrac_avg": 0.061473242938518524, + "val/num_eos_tokens": 748, + "val/ratio": 0.9898946285247803, + "val/ratio_var": 4.619114042725414e-05 + }, + { + "episode": 12752, + "epoch": 0.6440404040404041, + "eps": 0, + "loss/policy_avg": 0.06834741681814194, + "loss/value_avg": 0.5731651782989502, + "lr": 3.098930013082223e-06, + "objective/entropy": -88.31727600097656, + "objective/kl": 38.224117279052734, + "objective/non_score_reward": -1.9112060070037842, + "objective/rlhf_reward": 14.698169708251953, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008200570591725409, + "policy/clipfrac_avg": 0.007693252060562372, + "policy/entropy_avg": 0.1607208251953125, + "step": 797, + "val/clipfrac_avg": 0.11237858235836029, + "val/num_eos_tokens": 240, + "val/ratio": 0.9905504584312439, + "val/ratio_var": 4.706854815594852e-05 + }, + { + "episode": 12768, + "epoch": 0.6448484848484849, + "eps": 0, + "loss/policy_avg": -0.09761235117912292, + "loss/value_avg": 0.7088909149169922, + "lr": 3.0865828381745515e-06, + "objective/entropy": -124.7913589477539, + "objective/kl": 29.702392578125, + "objective/non_score_reward": -1.4851195812225342, + "objective/rlhf_reward": 13.68675422668457, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008782441727817059, + "policy/clipfrac_avg": 0.00598434591665864, + "policy/entropy_avg": 0.16924285888671875, + "step": 798, + "val/clipfrac_avg": 0.030641667544841766, + "val/num_eos_tokens": 454, + "val/ratio": 0.9882635474205017, + "val/ratio_var": 7.515518518630415e-05 + }, + { + "episode": 12784, + "epoch": 0.6456565656565657, + "eps": 0, + "loss/policy_avg": 0.14146611094474792, + "loss/value_avg": 0.7321012616157532, + "lr": 3.074249318355046e-06, + "objective/entropy": -101.5077133178711, + "objective/kl": 25.74894142150879, + "objective/non_score_reward": -1.2874470949172974, + "objective/rlhf_reward": 13.735990524291992, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007512210868299007, + "policy/clipfrac_avg": 0.011982694268226624, + "policy/entropy_avg": 0.15892601013183594, + "step": 799, + "val/clipfrac_avg": 0.012677992694079876, + "val/num_eos_tokens": 12, + "val/ratio": 0.9910134077072144, + "val/ratio_var": 6.09022390563041e-05 + }, + { + "episode": 12800, + "epoch": 0.6464646464646465, + "eps": 0, + "loss/policy_avg": -0.042851828038692474, + "loss/value_avg": 0.3452759385108948, + "lr": 3.0619295416417856e-06, + "objective/entropy": -68.88214874267578, + "objective/kl": 34.023399353027344, + "objective/non_score_reward": -1.7011699676513672, + "objective/rlhf_reward": 12.494142532348633, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010239251423627138, + "policy/clipfrac_avg": 0.010894561186432838, + "policy/entropy_avg": 0.17738628387451172, + "step": 800, + "val/clipfrac_avg": 0.026722928509116173, + "val/num_eos_tokens": 67, + "val/ratio": 0.9888595342636108, + "val/ratio_var": 6.888628558954224e-05 + }, + { + "episode": 12816, + "epoch": 0.6472727272727272, + "eps": 0, + "loss/policy_avg": -0.05754968896508217, + "loss/value_avg": 0.402332067489624, + "lr": 3.049623595954766e-06, + "objective/entropy": -89.62500762939453, + "objective/kl": 32.48891067504883, + "objective/non_score_reward": -1.6244455575942993, + "objective/rlhf_reward": 14.227116584777832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009385538287460804, + "policy/clipfrac_avg": 0.004920909181237221, + "policy/entropy_avg": 0.17440450191497803, + "step": 801, + "val/clipfrac_avg": 0.03176930546760559, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880223274230957, + "val/ratio_var": 5.388163845054805e-05 + }, + { + "episode": 12832, + "epoch": 0.648080808080808, + "eps": 0, + "loss/policy_avg": 0.2483101636171341, + "loss/value_avg": 0.23584425449371338, + "lr": 3.0373315691152856e-06, + "objective/entropy": -89.31193542480469, + "objective/kl": 40.699127197265625, + "objective/non_score_reward": -2.034956455230713, + "objective/rlhf_reward": 12.160355567932129, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0008679171442054212, + "policy/clipfrac_avg": 0.004822667222470045, + "policy/entropy_avg": 0.1660480499267578, + "step": 802, + "val/clipfrac_avg": 0.05813268944621086, + "val/num_eos_tokens": 149, + "val/ratio": 0.9890165328979492, + "val/ratio_var": 5.875104761798866e-05 + }, + { + "episode": 12848, + "epoch": 0.6488888888888888, + "eps": 0, + "loss/policy_avg": -0.10593897104263306, + "loss/value_avg": 0.5456434488296509, + "lr": 3.0250535488453077e-06, + "objective/entropy": -148.4324188232422, + "objective/kl": 38.51166915893555, + "objective/non_score_reward": -1.9255836009979248, + "objective/rlhf_reward": 11.918167114257812, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0007962615345604718, + "policy/clipfrac_avg": 0.0040261996909976006, + "policy/entropy_avg": 0.16094493865966797, + "step": 803, + "val/clipfrac_avg": 0.04293014109134674, + "val/num_eos_tokens": 331, + "val/ratio": 0.9893560409545898, + "val/ratio_var": 6.857667176518589e-05 + }, + { + "episode": 12864, + "epoch": 0.6496969696969697, + "eps": 0, + "loss/policy_avg": 0.06602153182029724, + "loss/value_avg": 0.2823004722595215, + "lr": 3.0127896227668367e-06, + "objective/entropy": -115.89801788330078, + "objective/kl": 40.921504974365234, + "objective/non_score_reward": -2.0460751056671143, + "objective/rlhf_reward": 12.867987632751465, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008664772612974048, + "policy/clipfrac_avg": 0.00966357346624136, + "policy/entropy_avg": 0.151214599609375, + "step": 804, + "val/clipfrac_avg": 0.03888742998242378, + "val/num_eos_tokens": 224, + "val/ratio": 0.9900601506233215, + "val/ratio_var": 5.00909227412194e-05 + }, + { + "episode": 12880, + "epoch": 0.6505050505050505, + "eps": 0, + "loss/policy_avg": -0.39892637729644775, + "loss/value_avg": 0.4240272641181946, + "lr": 3.000539878401296e-06, + "objective/entropy": -77.75199890136719, + "objective/kl": 35.056907653808594, + "objective/non_score_reward": -1.7528454065322876, + "objective/rlhf_reward": 14.00496768951416, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.001023960066959262, + "policy/clipfrac_avg": 0.004629913717508316, + "policy/entropy_avg": 0.20476627349853516, + "step": 805, + "val/clipfrac_avg": 0.07910472899675369, + "val/num_eos_tokens": 219, + "val/ratio": 0.9861233830451965, + "val/ratio_var": 8.373144373763353e-05 + }, + { + "episode": 12896, + "epoch": 0.6513131313131313, + "eps": 0, + "loss/policy_avg": -0.23323668539524078, + "loss/value_avg": 0.4607469439506531, + "lr": 2.9883044031688995e-06, + "objective/entropy": -85.76292419433594, + "objective/kl": 40.88092803955078, + "objective/non_score_reward": -2.044046401977539, + "objective/rlhf_reward": 14.174703598022461, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010081828804686666, + "policy/clipfrac_avg": 0.0041284579783678055, + "policy/entropy_avg": 0.1921253204345703, + "step": 806, + "val/clipfrac_avg": 0.16490784287452698, + "val/num_eos_tokens": 249, + "val/ratio": 0.9875838756561279, + "val/ratio_var": 6.450649379985407e-05 + }, + { + "episode": 12912, + "epoch": 0.6521212121212121, + "eps": 0, + "loss/policy_avg": -0.11845395714044571, + "loss/value_avg": 0.4592535197734833, + "lr": 2.976083284388031e-06, + "objective/entropy": -62.67930221557617, + "objective/kl": 33.47719955444336, + "objective/non_score_reward": -1.6738600730895996, + "objective/rlhf_reward": 12.771451950073242, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011091704946011305, + "policy/clipfrac_avg": 0.00747725460678339, + "policy/entropy_avg": 0.1983814239501953, + "step": 807, + "val/clipfrac_avg": 0.033114466816186905, + "val/num_eos_tokens": 17, + "val/ratio": 0.9877710938453674, + "val/ratio_var": 6.532426050398499e-05 + }, + { + "episode": 12928, + "epoch": 0.6529292929292929, + "eps": 0, + "loss/policy_avg": 0.0469900444149971, + "loss/value_avg": 0.26954370737075806, + "lr": 2.9638766092746207e-06, + "objective/entropy": -88.9464111328125, + "objective/kl": 34.25027847290039, + "objective/non_score_reward": -1.7125139236450195, + "objective/rlhf_reward": 12.56873607635498, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009448326891288161, + "policy/clipfrac_avg": 0.006387755274772644, + "policy/entropy_avg": 0.18566322326660156, + "step": 808, + "val/clipfrac_avg": 0.018177732825279236, + "val/num_eos_tokens": 761, + "val/ratio": 0.9882063269615173, + "val/ratio_var": 5.754105222877115e-05 + }, + { + "episode": 12944, + "epoch": 0.6537373737373737, + "eps": 0, + "loss/policy_avg": 0.00823296420276165, + "loss/value_avg": 0.44794195890426636, + "lr": 2.9516844649415165e-06, + "objective/entropy": -93.69944763183594, + "objective/kl": 40.86418151855469, + "objective/non_score_reward": -2.0432090759277344, + "objective/rlhf_reward": 13.542728424072266, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000799038156401366, + "policy/clipfrac_avg": 0.006408605724573135, + "policy/entropy_avg": 0.14969253540039062, + "step": 809, + "val/clipfrac_avg": 0.08976449072360992, + "val/num_eos_tokens": 215, + "val/ratio": 0.989780068397522, + "val/ratio_var": 6.0727201343979686e-05 + }, + { + "episode": 12960, + "epoch": 0.6545454545454545, + "eps": 0, + "loss/policy_avg": 0.024010583758354187, + "loss/value_avg": 0.3378852605819702, + "lr": 2.9395069383978725e-06, + "objective/entropy": -41.68758773803711, + "objective/kl": 35.73942184448242, + "objective/non_score_reward": -1.7869713306427002, + "objective/rlhf_reward": 15.314591407775879, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008846010314300656, + "policy/clipfrac_avg": 0.00692939106374979, + "policy/entropy_avg": 0.17317676544189453, + "step": 810, + "val/clipfrac_avg": 0.0032868448179215193, + "val/num_eos_tokens": 327, + "val/ratio": 0.9892865419387817, + "val/ratio_var": 4.3601932702586055e-05 + }, + { + "episode": 12976, + "epoch": 0.6553535353535354, + "eps": 0, + "loss/policy_avg": 0.1456170380115509, + "loss/value_avg": 0.5115708112716675, + "lr": 2.9273441165485227e-06, + "objective/entropy": -45.97090148925781, + "objective/kl": 36.01569366455078, + "objective/non_score_reward": -1.8007844686508179, + "objective/rlhf_reward": 14.23046588897705, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009762131376191974, + "policy/clipfrac_avg": 0.004803863354027271, + "policy/entropy_avg": 0.18830299377441406, + "step": 811, + "val/clipfrac_avg": 0.07713419198989868, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878898859024048, + "val/ratio_var": 4.7962184908101335e-05 + }, + { + "episode": 12992, + "epoch": 0.6561616161616162, + "eps": 0, + "loss/policy_avg": 0.1002146378159523, + "loss/value_avg": 0.558029294013977, + "lr": 2.9151960861933616e-06, + "objective/entropy": -66.14154052734375, + "objective/kl": 36.481346130371094, + "objective/non_score_reward": -1.8240673542022705, + "objective/rlhf_reward": 14.175932884216309, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009823704604059458, + "policy/clipfrac_avg": 0.0034461808390915394, + "policy/entropy_avg": 0.1955432891845703, + "step": 812, + "val/clipfrac_avg": 0.10748951137065887, + "val/num_eos_tokens": 292, + "val/ratio": 0.986059308052063, + "val/ratio_var": 5.328569022822194e-05 + }, + { + "episode": 13008, + "epoch": 0.656969696969697, + "eps": 0, + "loss/policy_avg": -0.10509809851646423, + "loss/value_avg": 0.46006062626838684, + "lr": 2.9030629340267165e-06, + "objective/entropy": -110.846923828125, + "objective/kl": 36.021034240722656, + "objective/non_score_reward": -1.8010516166687012, + "objective/rlhf_reward": 16.33176040649414, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008035163627937436, + "policy/clipfrac_avg": 0.005940265022218227, + "policy/entropy_avg": 0.15110206604003906, + "step": 813, + "val/clipfrac_avg": 0.07604548335075378, + "val/num_eos_tokens": 523, + "val/ratio": 0.9898133277893066, + "val/ratio_var": 6.0903592384420335e-05 + }, + { + "episode": 13024, + "epoch": 0.6577777777777778, + "eps": 0, + "loss/policy_avg": -0.11711955070495605, + "loss/value_avg": 1.2142528295516968, + "lr": 2.890944746636748e-06, + "objective/entropy": -40.75424575805664, + "objective/kl": 34.03726577758789, + "objective/non_score_reward": -1.701863408088684, + "objective/rlhf_reward": 13.091104507446289, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.000980376498773694, + "policy/clipfrac_avg": 0.008426186628639698, + "policy/entropy_avg": 0.20235157012939453, + "step": 814, + "val/clipfrac_avg": 0.02976190485060215, + "val/num_eos_tokens": 113, + "val/ratio": 0.9871314764022827, + "val/ratio_var": 6.27562912995927e-05 + }, + { + "episode": 13040, + "epoch": 0.6585858585858586, + "eps": 0, + "loss/policy_avg": -0.18143659830093384, + "loss/value_avg": 0.571576714515686, + "lr": 2.8788416105048124e-06, + "objective/entropy": -141.41366577148438, + "objective/kl": 42.53788757324219, + "objective/non_score_reward": -2.126894474029541, + "objective/rlhf_reward": 13.537168502807617, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008768361294642091, + "policy/clipfrac_avg": 0.006732264067977667, + "policy/entropy_avg": 0.17724037170410156, + "step": 815, + "val/clipfrac_avg": 0.03276197984814644, + "val/num_eos_tokens": 221, + "val/ratio": 0.9885480403900146, + "val/ratio_var": 8.229191007558256e-05 + }, + { + "episode": 13056, + "epoch": 0.6593939393939394, + "eps": 0, + "loss/policy_avg": 0.3017633855342865, + "loss/value_avg": 0.32543864846229553, + "lr": 2.8667536120048616e-06, + "objective/entropy": -61.10264587402344, + "objective/kl": 35.6143684387207, + "objective/non_score_reward": -1.7807183265686035, + "objective/rlhf_reward": 14.391157150268555, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009194887243211269, + "policy/clipfrac_avg": 0.018624408170580864, + "policy/entropy_avg": 0.14903783798217773, + "step": 816, + "val/clipfrac_avg": 0.0073128631338477135, + "val/num_eos_tokens": 112, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.833535033161752e-05 + }, + { + "episode": 13072, + "epoch": 0.6602020202020202, + "eps": 0, + "loss/policy_avg": 0.06899265199899673, + "loss/value_avg": 0.6989455223083496, + "lr": 2.854680837402804e-06, + "objective/entropy": -117.86632537841797, + "objective/kl": 34.15174102783203, + "objective/non_score_reward": -1.7075870037078857, + "objective/rlhf_reward": 14.511162757873535, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009505614871159196, + "policy/clipfrac_avg": 0.008451011963188648, + "policy/entropy_avg": 0.1699972152709961, + "step": 817, + "val/clipfrac_avg": 0.000589622650295496, + "val/num_eos_tokens": 435, + "val/ratio": 0.9876314401626587, + "val/ratio_var": 9.73327478277497e-05 + }, + { + "episode": 13088, + "epoch": 0.661010101010101, + "eps": 0, + "loss/policy_avg": -0.05428570508956909, + "loss/value_avg": 0.2928343415260315, + "lr": 2.8426233728559172e-06, + "objective/entropy": -52.28473663330078, + "objective/kl": 40.512550354003906, + "objective/non_score_reward": -2.025627613067627, + "objective/rlhf_reward": 14.458747863769531, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008845083648338914, + "policy/clipfrac_avg": 0.005094105843454599, + "policy/entropy_avg": 0.16685867309570312, + "step": 818, + "val/clipfrac_avg": 0.09271319210529327, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885064959526062, + "val/ratio_var": 4.853576319874264e-05 + }, + { + "episode": 13104, + "epoch": 0.6618181818181819, + "eps": 0, + "loss/policy_avg": 0.12085426598787308, + "loss/value_avg": 0.5670195817947388, + "lr": 2.83058130441221e-06, + "objective/entropy": -196.3238983154297, + "objective/kl": 42.012672424316406, + "objective/non_score_reward": -2.1006336212158203, + "objective/rlhf_reward": 12.51655387878418, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007904545636847615, + "policy/clipfrac_avg": 0.006419865880161524, + "policy/entropy_avg": 0.1437664031982422, + "step": 819, + "val/clipfrac_avg": 0.05388574302196503, + "val/num_eos_tokens": 496, + "val/ratio": 0.9899193644523621, + "val/ratio_var": 4.292546145734377e-05 + }, + { + "episode": 13120, + "epoch": 0.6626262626262627, + "eps": 0, + "loss/policy_avg": 0.6058844327926636, + "loss/value_avg": 0.7294787168502808, + "lr": 2.8185547180098195e-06, + "objective/entropy": -58.513553619384766, + "objective/kl": 38.73785400390625, + "objective/non_score_reward": -1.9368927478790283, + "objective/rlhf_reward": 12.883419036865234, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010288474150002003, + "policy/clipfrac_avg": 0.021498143672943115, + "policy/entropy_avg": 0.20896148681640625, + "step": 820, + "val/clipfrac_avg": 0.0723545029759407, + "val/num_eos_tokens": 134, + "val/ratio": 0.9887969493865967, + "val/ratio_var": 4.8135509132407606e-05 + }, + { + "episode": 13136, + "epoch": 0.6634343434343435, + "eps": 0, + "loss/policy_avg": -0.21932660043239594, + "loss/value_avg": 0.45023149251937866, + "lr": 2.8065436994763963e-06, + "objective/entropy": -38.43750762939453, + "objective/kl": 33.36050033569336, + "objective/non_score_reward": -1.6680251359939575, + "objective/rlhf_reward": 13.542912483215332, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010698122205212712, + "policy/clipfrac_avg": 0.004931025672703981, + "policy/entropy_avg": 0.19058465957641602, + "step": 821, + "val/clipfrac_avg": 0.00852490495890379, + "val/num_eos_tokens": 190, + "val/ratio": 0.9864101409912109, + "val/ratio_var": 5.521172352018766e-05 + }, + { + "episode": 13152, + "epoch": 0.6642424242424242, + "eps": 0, + "loss/policy_avg": -0.01703064516186714, + "loss/value_avg": 0.2902008295059204, + "lr": 2.794548334528486e-06, + "objective/entropy": 13.10763931274414, + "objective/kl": 44.57916259765625, + "objective/non_score_reward": -2.2289581298828125, + "objective/rlhf_reward": 13.364792823791504, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0014038155786693096, + "policy/clipfrac_avg": 0.006509849801659584, + "policy/entropy_avg": 0.26366424560546875, + "step": 822, + "val/clipfrac_avg": 0.03128276392817497, + "val/num_eos_tokens": 35, + "val/ratio": 0.9824488162994385, + "val/ratio_var": 4.7705398174002767e-05 + }, + { + "episode": 13168, + "epoch": 0.665050505050505, + "eps": 0, + "loss/policy_avg": -0.02819274365901947, + "loss/value_avg": 0.2861873507499695, + "lr": 2.782568708770933e-06, + "objective/entropy": -94.84840393066406, + "objective/kl": 42.75294494628906, + "objective/non_score_reward": -2.1376476287841797, + "objective/rlhf_reward": 12.26860237121582, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010491260327398777, + "policy/clipfrac_avg": 0.004985281266272068, + "policy/entropy_avg": 0.2000885009765625, + "step": 823, + "val/clipfrac_avg": 0.00754884909838438, + "val/num_eos_tokens": 289, + "val/ratio": 0.9866741895675659, + "val/ratio_var": 4.918476770399138e-05 + }, + { + "episode": 13184, + "epoch": 0.6658585858585858, + "eps": 0, + "loss/policy_avg": -0.024818487465381622, + "loss/value_avg": 0.3611629605293274, + "lr": 2.7706049076962483e-06, + "objective/entropy": -104.7535400390625, + "objective/kl": 45.08838653564453, + "objective/non_score_reward": -2.2544193267822266, + "objective/rlhf_reward": 12.214330673217773, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009155337465927005, + "policy/clipfrac_avg": 0.008831838145852089, + "policy/entropy_avg": 0.16748523712158203, + "step": 824, + "val/clipfrac_avg": 0.0024258445482701063, + "val/num_eos_tokens": 163, + "val/ratio": 0.9890375137329102, + "val/ratio_var": 5.131318175699562e-05 + }, + { + "episode": 13200, + "epoch": 0.6666666666666666, + "eps": 0, + "loss/policy_avg": -0.28224268555641174, + "loss/value_avg": 0.8720104694366455, + "lr": 2.7586570166840154e-06, + "objective/entropy": -74.55836486816406, + "objective/kl": 30.59856605529785, + "objective/non_score_reward": -1.52992844581604, + "objective/rlhf_reward": 15.329446792602539, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008330290438607335, + "policy/clipfrac_avg": 0.010464916005730629, + "policy/entropy_avg": 0.15100550651550293, + "step": 825, + "val/clipfrac_avg": 0.08580771088600159, + "val/num_eos_tokens": 460, + "val/ratio": 0.9896235466003418, + "val/ratio_var": 9.328931628260761e-05 + }, + { + "episode": 13216, + "epoch": 0.6674747474747474, + "eps": 0, + "loss/policy_avg": -0.1715128868818283, + "loss/value_avg": 0.4540337920188904, + "lr": 2.746725121000273e-06, + "objective/entropy": -43.89616394042969, + "objective/kl": 41.760223388671875, + "objective/non_score_reward": -2.0880112648010254, + "objective/rlhf_reward": 13.872926712036133, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010317859705537558, + "policy/clipfrac_avg": 0.0054151020012795925, + "policy/entropy_avg": 0.19500446319580078, + "step": 826, + "val/clipfrac_avg": 0.03631855547428131, + "val/num_eos_tokens": 334, + "val/ratio": 0.9874616265296936, + "val/ratio_var": 6.214805034687743e-05 + }, + { + "episode": 13232, + "epoch": 0.6682828282828283, + "eps": 0, + "loss/policy_avg": -0.077702596783638, + "loss/value_avg": 0.6576616168022156, + "lr": 2.734809305796915e-06, + "objective/entropy": -13.071683883666992, + "objective/kl": 34.05264663696289, + "objective/non_score_reward": -1.7026324272155762, + "objective/rlhf_reward": 14.656742095947266, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0012533586705103517, + "policy/clipfrac_avg": 0.00517519935965538, + "policy/entropy_avg": 0.21735763549804688, + "step": 827, + "val/clipfrac_avg": 0.09026823937892914, + "val/num_eos_tokens": 468, + "val/ratio": 0.9863014221191406, + "val/ratio_var": 0.00010488701082067564 + }, + { + "episode": 13248, + "epoch": 0.6690909090909091, + "eps": 0, + "loss/policy_avg": -0.11604561656713486, + "loss/value_avg": 0.18491807579994202, + "lr": 2.7229096561110703e-06, + "objective/entropy": -132.30560302734375, + "objective/kl": 37.0118522644043, + "objective/non_score_reward": -1.8505926132202148, + "objective/rlhf_reward": 15.282218933105469, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008336640894412994, + "policy/clipfrac_avg": 0.0030343476682901382, + "policy/entropy_avg": 0.16221046447753906, + "step": 828, + "val/clipfrac_avg": 0.07020965218544006, + "val/num_eos_tokens": 41, + "val/ratio": 0.9894965291023254, + "val/ratio_var": 4.505051037995145e-05 + }, + { + "episode": 13264, + "epoch": 0.6698989898989899, + "eps": 0, + "loss/policy_avg": -0.23082350194454193, + "loss/value_avg": 0.9338909387588501, + "lr": 2.7110262568645063e-06, + "objective/entropy": -83.5711441040039, + "objective/kl": 42.80398941040039, + "objective/non_score_reward": -2.1401994228363037, + "objective/rlhf_reward": 13.555112838745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008490884792990983, + "policy/clipfrac_avg": 0.024211544543504715, + "policy/entropy_avg": 0.16208171844482422, + "step": 829, + "val/clipfrac_avg": 0.01703277789056301, + "val/num_eos_tokens": 407, + "val/ratio": 0.9889751672744751, + "val/ratio_var": 6.549843965331092e-05 + }, + { + "episode": 13280, + "epoch": 0.6707070707070707, + "eps": 0, + "loss/policy_avg": -0.1790890097618103, + "loss/value_avg": 0.9402360320091248, + "lr": 2.699159192863018e-06, + "objective/entropy": -91.52725219726562, + "objective/kl": 30.28129005432129, + "objective/non_score_reward": -1.5140645503997803, + "objective/rlhf_reward": 13.71249771118164, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0006465939804911613, + "policy/clipfrac_avg": 0.003674568608403206, + "policy/entropy_avg": 0.14063799381256104, + "step": 830, + "val/clipfrac_avg": 0.05747377499938011, + "val/num_eos_tokens": 250, + "val/ratio": 0.9919025897979736, + "val/ratio_var": 6.961268809391186e-05 + }, + { + "episode": 13296, + "epoch": 0.6715151515151515, + "eps": 0, + "loss/policy_avg": 0.3400639295578003, + "loss/value_avg": 1.9688762426376343, + "lr": 2.687308548795825e-06, + "objective/entropy": -54.39606475830078, + "objective/kl": 44.12242889404297, + "objective/non_score_reward": -2.2061214447021484, + "objective/rlhf_reward": 14.004816055297852, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009418983245268464, + "policy/clipfrac_avg": 0.009523827582597733, + "policy/entropy_avg": 0.18965661525726318, + "step": 831, + "val/clipfrac_avg": 0.045346200466156006, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880996942520142, + "val/ratio_var": 6.565819057868794e-05 + }, + { + "episode": 13312, + "epoch": 0.6723232323232323, + "eps": 0, + "loss/policy_avg": 0.11609108746051788, + "loss/value_avg": 0.5635161399841309, + "lr": 2.675474409234965e-06, + "objective/entropy": -66.36386108398438, + "objective/kl": 38.93806838989258, + "objective/non_score_reward": -1.9469034671783447, + "objective/rlhf_reward": 14.076534271240234, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008638559957034886, + "policy/clipfrac_avg": 0.005322727840393782, + "policy/entropy_avg": 0.15152549743652344, + "step": 832, + "val/clipfrac_avg": 0.04381619766354561, + "val/num_eos_tokens": 20, + "val/ratio": 0.9889363050460815, + "val/ratio_var": 3.002676021424122e-05 + }, + { + "episode": 13328, + "epoch": 0.6731313131313131, + "eps": 0, + "loss/policy_avg": -0.03308400511741638, + "loss/value_avg": 0.3210276961326599, + "lr": 2.66365685863469e-06, + "objective/entropy": -111.39958953857422, + "objective/kl": 47.16798400878906, + "objective/non_score_reward": -2.3583993911743164, + "objective/rlhf_reward": 11.7197265625, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009453936363570392, + "policy/clipfrac_avg": 0.004603059496730566, + "policy/entropy_avg": 0.18264532089233398, + "step": 833, + "val/clipfrac_avg": 0.062541663646698, + "val/num_eos_tokens": 763, + "val/ratio": 0.9879401922225952, + "val/ratio_var": 6.487595965154469e-05 + }, + { + "episode": 13344, + "epoch": 0.673939393939394, + "eps": 0, + "loss/policy_avg": 0.051825061440467834, + "loss/value_avg": 0.8246160745620728, + "lr": 2.651855981330872e-06, + "objective/entropy": -69.66492462158203, + "objective/kl": 37.68748474121094, + "objective/non_score_reward": -1.8843741416931152, + "objective/rlhf_reward": 12.764063835144043, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010055399034172297, + "policy/clipfrac_avg": 0.004349449649453163, + "policy/entropy_avg": 0.18214893341064453, + "step": 834, + "val/clipfrac_avg": 0.06374436616897583, + "val/num_eos_tokens": 38, + "val/ratio": 0.9867125749588013, + "val/ratio_var": 9.083253826247528e-05 + }, + { + "episode": 13360, + "epoch": 0.6747474747474748, + "eps": 0, + "loss/policy_avg": -0.043558381497859955, + "loss/value_avg": 0.6093795895576477, + "lr": 2.6400718615403852e-06, + "objective/entropy": -108.79554748535156, + "objective/kl": 39.288482666015625, + "objective/non_score_reward": -1.9644241333007812, + "objective/rlhf_reward": 14.504325866699219, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010659170802682638, + "policy/clipfrac_avg": 0.005707671865820885, + "policy/entropy_avg": 0.1986398696899414, + "step": 835, + "val/clipfrac_avg": 0.013371335342526436, + "val/num_eos_tokens": 647, + "val/ratio": 0.9869523048400879, + "val/ratio_var": 8.708865061635152e-05 + }, + { + "episode": 13376, + "epoch": 0.6755555555555556, + "eps": 0, + "loss/policy_avg": 0.09040576219558716, + "loss/value_avg": 0.4517725706100464, + "lr": 2.62830458336052e-06, + "objective/entropy": -59.1908073425293, + "objective/kl": 42.06044387817383, + "objective/non_score_reward": -2.1030220985412598, + "objective/rlhf_reward": 13.842290878295898, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001016529044136405, + "policy/clipfrac_avg": 0.007354079745709896, + "policy/entropy_avg": 0.20728302001953125, + "step": 836, + "val/clipfrac_avg": 0.030187882483005524, + "val/num_eos_tokens": 477, + "val/ratio": 0.9865027666091919, + "val/ratio_var": 5.405073534348048e-05 + }, + { + "episode": 13392, + "epoch": 0.6763636363636364, + "eps": 0, + "loss/policy_avg": -0.21371892094612122, + "loss/value_avg": 0.26992350816726685, + "lr": 2.6165542307683744e-06, + "objective/entropy": -99.01640319824219, + "objective/kl": 43.636474609375, + "objective/non_score_reward": -2.18182373046875, + "objective/rlhf_reward": 13.08380126953125, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009176525054499507, + "policy/clipfrac_avg": 0.006078221835196018, + "policy/entropy_avg": 0.1714763641357422, + "step": 837, + "val/clipfrac_avg": 0.025651777163147926, + "val/num_eos_tokens": 453, + "val/ratio": 0.988213300704956, + "val/ratio_var": 6.253684114199132e-05 + }, + { + "episode": 13408, + "epoch": 0.6771717171717172, + "eps": 0, + "loss/policy_avg": 0.08225658535957336, + "loss/value_avg": 0.4106031656265259, + "lr": 2.604820887620254e-06, + "objective/entropy": -62.21269989013672, + "objective/kl": 40.305843353271484, + "objective/non_score_reward": -2.015292167663574, + "objective/rlhf_reward": 15.75814437866211, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0009111049002967775, + "policy/clipfrac_avg": 0.005017759278416634, + "policy/entropy_avg": 0.18220901489257812, + "step": 838, + "val/clipfrac_avg": 0.0028512999415397644, + "val/num_eos_tokens": 268, + "val/ratio": 0.9878619909286499, + "val/ratio_var": 3.237010969314724e-05 + }, + { + "episode": 13424, + "epoch": 0.677979797979798, + "eps": 0, + "loss/policy_avg": 0.01355072483420372, + "loss/value_avg": 0.4876829981803894, + "lr": 2.5931046376510875e-06, + "objective/entropy": -38.304931640625, + "objective/kl": 35.24858093261719, + "objective/non_score_reward": -1.7624289989471436, + "objective/rlhf_reward": 12.284445762634277, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0008708706009201705, + "policy/clipfrac_avg": 0.005640837829560041, + "policy/entropy_avg": 0.16802191734313965, + "step": 839, + "val/clipfrac_avg": 0.029899097979068756, + "val/num_eos_tokens": 258, + "val/ratio": 0.989201009273529, + "val/ratio_var": 5.1689257816178724e-05 + }, + { + "episode": 13440, + "epoch": 0.6787878787878788, + "eps": 0, + "loss/policy_avg": 0.1623387336730957, + "loss/value_avg": 0.3584904074668884, + "lr": 2.5814055644738013e-06, + "objective/entropy": -63.923397064208984, + "objective/kl": 36.323509216308594, + "objective/non_score_reward": -1.8161752223968506, + "objective/rlhf_reward": 12.97288703918457, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009589061955921352, + "policy/clipfrac_avg": 0.006033012643456459, + "policy/entropy_avg": 0.17229270935058594, + "step": 840, + "val/clipfrac_avg": 0.11569125950336456, + "val/num_eos_tokens": 159, + "val/ratio": 0.988390326499939, + "val/ratio_var": 7.233954966068268e-05 + }, + { + "episode": 13456, + "epoch": 0.6795959595959596, + "eps": 0, + "loss/policy_avg": -0.29571330547332764, + "loss/value_avg": 0.5451829433441162, + "lr": 2.5697237515787564e-06, + "objective/entropy": -159.0351104736328, + "objective/kl": 34.90948486328125, + "objective/non_score_reward": -1.7454744577407837, + "objective/rlhf_reward": 13.145150184631348, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0007017417810857296, + "policy/clipfrac_avg": 0.005396063905209303, + "policy/entropy_avg": 0.12675732374191284, + "step": 841, + "val/clipfrac_avg": 0.019056372344493866, + "val/num_eos_tokens": 508, + "val/ratio": 0.9914215207099915, + "val/ratio_var": 8.740482735447586e-05 + }, + { + "episode": 13472, + "epoch": 0.6804040404040405, + "eps": 0, + "loss/policy_avg": -0.33980700373649597, + "loss/value_avg": 0.5796597003936768, + "lr": 2.5580592823331237e-06, + "objective/entropy": -83.5593490600586, + "objective/kl": 36.281490325927734, + "objective/non_score_reward": -1.8140745162963867, + "objective/rlhf_reward": 14.18592643737793, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009474118705838919, + "policy/clipfrac_avg": 0.004876702558249235, + "policy/entropy_avg": 0.14034080505371094, + "step": 842, + "val/clipfrac_avg": 0.03151373565196991, + "val/num_eos_tokens": 208, + "val/ratio": 0.9901444911956787, + "val/ratio_var": 6.344315625028685e-05 + }, + { + "episode": 13488, + "epoch": 0.6812121212121212, + "eps": 0, + "loss/policy_avg": 0.11847849935293198, + "loss/value_avg": 0.3053319454193115, + "lr": 2.5464122399803126e-06, + "objective/entropy": -74.54005432128906, + "objective/kl": 46.23011779785156, + "objective/non_score_reward": -2.3115057945251465, + "objective/rlhf_reward": 12.993181228637695, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009884861065074801, + "policy/clipfrac_avg": 0.006667924579232931, + "policy/entropy_avg": 0.19658470153808594, + "step": 843, + "val/clipfrac_avg": 0.00010080645006382838, + "val/num_eos_tokens": 52, + "val/ratio": 0.9878820180892944, + "val/ratio_var": 4.166604412603192e-05 + }, + { + "episode": 13504, + "epoch": 0.682020202020202, + "eps": 0, + "loss/policy_avg": 0.07477365434169769, + "loss/value_avg": 0.717211902141571, + "lr": 2.5347827076393505e-06, + "objective/entropy": -55.136016845703125, + "objective/kl": 36.799835205078125, + "objective/non_score_reward": -1.8399916887283325, + "objective/rlhf_reward": 12.995945930480957, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010256943060085177, + "policy/clipfrac_avg": 0.009538116864860058, + "policy/entropy_avg": 0.19795989990234375, + "step": 844, + "val/clipfrac_avg": 0.0769520252943039, + "val/num_eos_tokens": 573, + "val/ratio": 0.9861817955970764, + "val/ratio_var": 7.031750283204019e-05 + }, + { + "episode": 13520, + "epoch": 0.6828282828282828, + "eps": 0, + "loss/policy_avg": -0.010380454361438751, + "loss/value_avg": 0.8978620767593384, + "lr": 2.5231707683043204e-06, + "objective/entropy": -164.994873046875, + "objective/kl": 41.09418487548828, + "objective/non_score_reward": -2.0547094345092773, + "objective/rlhf_reward": 12.367165565490723, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006862258305773139, + "policy/clipfrac_avg": 0.005633420776575804, + "policy/entropy_avg": 0.136183500289917, + "step": 845, + "val/clipfrac_avg": 0.10761284828186035, + "val/num_eos_tokens": 289, + "val/ratio": 0.9907996654510498, + "val/ratio_var": 4.169603198533878e-05 + }, + { + "episode": 13536, + "epoch": 0.6836363636363636, + "eps": 0, + "loss/policy_avg": -0.0011942088603973389, + "loss/value_avg": 0.7805696129798889, + "lr": 2.5115765048437445e-06, + "objective/entropy": -119.26876068115234, + "objective/kl": 37.73551940917969, + "objective/non_score_reward": -1.8867759704589844, + "objective/rlhf_reward": 13.769474029541016, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0007093651220202446, + "policy/clipfrac_avg": 0.028275154531002045, + "policy/entropy_avg": 0.1421506404876709, + "step": 846, + "val/clipfrac_avg": 0.08967090398073196, + "val/num_eos_tokens": 760, + "val/ratio": 0.9911968111991882, + "val/ratio_var": 3.498927253531292e-05 + }, + { + "episode": 13552, + "epoch": 0.6844444444444444, + "eps": 0, + "loss/policy_avg": -0.007019490003585815, + "loss/value_avg": 0.8198627233505249, + "lr": 2.5000000000000015e-06, + "objective/entropy": -89.728271484375, + "objective/kl": 34.76280975341797, + "objective/non_score_reward": -1.7381404638290405, + "objective/rlhf_reward": 13.925922393798828, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009024562314152718, + "policy/clipfrac_avg": 0.007501127198338509, + "policy/entropy_avg": 0.17042064666748047, + "step": 847, + "val/clipfrac_avg": 0.028372831642627716, + "val/num_eos_tokens": 17, + "val/ratio": 0.9899512529373169, + "val/ratio_var": 7.211479532998055e-05 + }, + { + "episode": 13568, + "epoch": 0.6852525252525252, + "eps": 0, + "loss/policy_avg": 0.015100933611392975, + "loss/value_avg": 0.25284868478775024, + "lr": 2.488441336388737e-06, + "objective/entropy": -184.20425415039062, + "objective/kl": 46.20475769042969, + "objective/non_score_reward": -2.3102383613586426, + "objective/rlhf_reward": 11.728824615478516, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007638686802238226, + "policy/clipfrac_avg": 0.005230149254202843, + "policy/entropy_avg": 0.15384864807128906, + "step": 848, + "val/clipfrac_avg": 0.01555095985531807, + "val/num_eos_tokens": 404, + "val/ratio": 0.9898365139961243, + "val/ratio_var": 5.412843165686354e-05 + }, + { + "episode": 13584, + "epoch": 0.686060606060606, + "eps": 0, + "loss/policy_avg": -0.13074953854084015, + "loss/value_avg": 0.5254482626914978, + "lr": 2.4769005964982718e-06, + "objective/entropy": -70.96700286865234, + "objective/kl": 37.31745147705078, + "objective/non_score_reward": -1.8658727407455444, + "objective/rlhf_reward": 13.665377616882324, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000885509536601603, + "policy/clipfrac_avg": 0.006637534126639366, + "policy/entropy_avg": 0.17204856872558594, + "step": 849, + "val/clipfrac_avg": 0.050514861941337585, + "val/num_eos_tokens": 24, + "val/ratio": 0.9882212281227112, + "val/ratio_var": 5.447021248983219e-05 + }, + { + "episode": 13600, + "epoch": 0.6868686868686869, + "eps": 0, + "loss/policy_avg": 0.039310187101364136, + "loss/value_avg": 0.18585552275180817, + "lr": 2.465377862689019e-06, + "objective/entropy": -62.67339324951172, + "objective/kl": 48.39326477050781, + "objective/non_score_reward": -2.419663190841675, + "objective/rlhf_reward": 13.564711570739746, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009431451326236129, + "policy/clipfrac_avg": 0.004864457994699478, + "policy/entropy_avg": 0.19204330444335938, + "step": 850, + "val/clipfrac_avg": 0.01945144683122635, + "val/num_eos_tokens": 36, + "val/ratio": 0.9872653484344482, + "val/ratio_var": 3.194732926203869e-05 + }, + { + "episode": 13616, + "epoch": 0.6876767676767677, + "eps": 0, + "loss/policy_avg": 0.07975710183382034, + "loss/value_avg": 0.478973388671875, + "lr": 2.4538732171928847e-06, + "objective/entropy": -76.03438568115234, + "objective/kl": 35.364593505859375, + "objective/non_score_reward": -1.7682294845581055, + "objective/rlhf_reward": 15.075519561767578, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008544789161533117, + "policy/clipfrac_avg": 0.005153194069862366, + "policy/entropy_avg": 0.16641998291015625, + "step": 851, + "val/clipfrac_avg": 0.03764945641160011, + "val/num_eos_tokens": 298, + "val/ratio": 0.9884979724884033, + "val/ratio_var": 4.7100515075726435e-05 + }, + { + "episode": 13632, + "epoch": 0.6884848484848485, + "eps": 0, + "loss/policy_avg": -0.10211583226919174, + "loss/value_avg": 0.7199238538742065, + "lr": 2.4423867421126923e-06, + "objective/entropy": -59.744590759277344, + "objective/kl": 35.15203857421875, + "objective/non_score_reward": -1.7576020956039429, + "objective/rlhf_reward": 15.05489730834961, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009704260155558586, + "policy/clipfrac_avg": 0.004688786808401346, + "policy/entropy_avg": 0.17905235290527344, + "step": 852, + "val/clipfrac_avg": 0.17230291664600372, + "val/num_eos_tokens": 644, + "val/ratio": 0.9872374534606934, + "val/ratio_var": 4.29930878453888e-05 + }, + { + "episode": 13648, + "epoch": 0.6892929292929293, + "eps": 0, + "loss/policy_avg": 0.0028443392366170883, + "loss/value_avg": 0.3421381115913391, + "lr": 2.4309185194215885e-06, + "objective/entropy": -63.859352111816406, + "objective/kl": 41.87160110473633, + "objective/non_score_reward": -2.0935800075531006, + "objective/rlhf_reward": 16.01579475402832, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0011341955978423357, + "policy/clipfrac_avg": 0.005558031145483255, + "policy/entropy_avg": 0.21439552307128906, + "step": 853, + "val/clipfrac_avg": 0.07718860357999802, + "val/num_eos_tokens": 400, + "val/ratio": 0.9858847856521606, + "val/ratio_var": 5.448433512356132e-05 + }, + { + "episode": 13664, + "epoch": 0.6901010101010101, + "eps": 0, + "loss/policy_avg": 0.11498575657606125, + "loss/value_avg": 0.909265398979187, + "lr": 2.4194686309624664e-06, + "objective/entropy": -138.73922729492188, + "objective/kl": 39.14732360839844, + "objective/non_score_reward": -1.9573662281036377, + "objective/rlhf_reward": 11.542634010314941, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0011413603788241744, + "policy/clipfrac_avg": 0.008165586739778519, + "policy/entropy_avg": 0.18193864822387695, + "step": 854, + "val/clipfrac_avg": 0.04511754959821701, + "val/num_eos_tokens": 264, + "val/ratio": 0.9880124926567078, + "val/ratio_var": 7.85917291068472e-05 + }, + { + "episode": 13680, + "epoch": 0.6909090909090909, + "eps": 0, + "loss/policy_avg": 0.4261026978492737, + "loss/value_avg": 1.0351085662841797, + "lr": 2.408037158447375e-06, + "objective/entropy": -80.8604736328125, + "objective/kl": 39.87467956542969, + "objective/non_score_reward": -1.9937340021133423, + "objective/rlhf_reward": 12.850015640258789, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000854466634336859, + "policy/clipfrac_avg": 0.0031361770816147327, + "policy/entropy_avg": 0.1807117462158203, + "step": 855, + "val/clipfrac_avg": 0.06461638957262039, + "val/num_eos_tokens": 76, + "val/ratio": 0.9892903566360474, + "val/ratio_var": 6.158453470561653e-05 + }, + { + "episode": 13696, + "epoch": 0.6917171717171717, + "eps": 0, + "loss/policy_avg": 0.35515934228897095, + "loss/value_avg": 0.6004530787467957, + "lr": 2.3966241834569364e-06, + "objective/entropy": -109.00687408447266, + "objective/kl": 44.59175109863281, + "objective/non_score_reward": -2.2295875549316406, + "objective/rlhf_reward": 14.02041244506836, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008301680209115148, + "policy/clipfrac_avg": 0.004295141436159611, + "policy/entropy_avg": 0.16374778747558594, + "step": 856, + "val/clipfrac_avg": 0.0578775517642498, + "val/num_eos_tokens": 355, + "val/ratio": 0.9892109632492065, + "val/ratio_var": 3.649172867881134e-05 + }, + { + "episode": 13712, + "epoch": 0.6925252525252525, + "eps": 0, + "loss/policy_avg": -0.07683120667934418, + "loss/value_avg": 0.7507062554359436, + "lr": 2.385229787439768e-06, + "objective/entropy": -65.75347137451172, + "objective/kl": 33.233463287353516, + "objective/non_score_reward": -1.6616731882095337, + "objective/rlhf_reward": 12.885202407836914, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008445363491773605, + "policy/clipfrac_avg": 0.0028887931257486343, + "policy/entropy_avg": 0.1688997745513916, + "step": 857, + "val/clipfrac_avg": 0.029584139585494995, + "val/num_eos_tokens": 79, + "val/ratio": 0.9893004894256592, + "val/ratio_var": 4.4566000724444166e-05 + }, + { + "episode": 13728, + "epoch": 0.6933333333333334, + "eps": 0, + "loss/policy_avg": -0.04102202132344246, + "loss/value_avg": 0.295141339302063, + "lr": 2.3738540517118953e-06, + "objective/entropy": -137.21546936035156, + "objective/kl": 39.995941162109375, + "objective/non_score_reward": -1.9997971057891846, + "objective/rlhf_reward": 12.875202178955078, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009317261283285916, + "policy/clipfrac_avg": 0.016368409618735313, + "policy/entropy_avg": 0.18145465850830078, + "step": 858, + "val/clipfrac_avg": 0.02603679709136486, + "val/num_eos_tokens": 133, + "val/ratio": 0.9879333972930908, + "val/ratio_var": 7.448039104929194e-05 + }, + { + "episode": 13744, + "epoch": 0.6941414141414142, + "eps": 0, + "loss/policy_avg": -0.09325989335775375, + "loss/value_avg": 0.4778023660182953, + "lr": 2.3624970574561773e-06, + "objective/entropy": -126.39140319824219, + "objective/kl": 41.33527755737305, + "objective/non_score_reward": -2.0667638778686523, + "objective/rlhf_reward": 15.823860168457031, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009032993111759424, + "policy/clipfrac_avg": 0.008262147195637226, + "policy/entropy_avg": 0.1809234619140625, + "step": 859, + "val/clipfrac_avg": 0.006358179729431868, + "val/num_eos_tokens": 198, + "val/ratio": 0.9881711602210999, + "val/ratio_var": 8.430143498117104e-05 + }, + { + "episode": 13760, + "epoch": 0.694949494949495, + "eps": 0, + "loss/policy_avg": -0.1992878019809723, + "loss/value_avg": 0.4450816214084625, + "lr": 2.3511588857217228e-06, + "objective/entropy": -82.32437133789062, + "objective/kl": 36.61549377441406, + "objective/non_score_reward": -1.8307745456695557, + "objective/rlhf_reward": 14.497350692749023, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009405683958902955, + "policy/clipfrac_avg": 0.0031182412058115005, + "policy/entropy_avg": 0.1852397918701172, + "step": 860, + "val/clipfrac_avg": 0.04030741751194, + "val/num_eos_tokens": 674, + "val/ratio": 0.9874944686889648, + "val/ratio_var": 5.9595575294224545e-05 + }, + { + "episode": 13776, + "epoch": 0.6957575757575758, + "eps": 0, + "loss/policy_avg": -0.0025226399302482605, + "loss/value_avg": 0.6712653636932373, + "lr": 2.339839617423318e-06, + "objective/entropy": -97.76516723632812, + "objective/kl": 32.36253356933594, + "objective/non_score_reward": -1.6181267499923706, + "objective/rlhf_reward": 14.108436584472656, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008775125024840236, + "policy/clipfrac_avg": 0.006763250567018986, + "policy/entropy_avg": 0.15996170043945312, + "step": 861, + "val/clipfrac_avg": 0.01572282984852791, + "val/num_eos_tokens": 755, + "val/ratio": 0.9897935390472412, + "val/ratio_var": 4.971764428773895e-05 + }, + { + "episode": 13792, + "epoch": 0.6965656565656566, + "eps": 0, + "loss/policy_avg": 0.08292508125305176, + "loss/value_avg": 0.5537270307540894, + "lr": 2.32853933334084e-06, + "objective/entropy": -61.261680603027344, + "objective/kl": 36.61561584472656, + "objective/non_score_reward": -1.8307808637619019, + "objective/rlhf_reward": 12.762969970703125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009761417168192565, + "policy/clipfrac_avg": 0.0026826506946235895, + "policy/entropy_avg": 0.22123146057128906, + "step": 862, + "val/clipfrac_avg": 0.044523607939481735, + "val/num_eos_tokens": 56, + "val/ratio": 0.9871484041213989, + "val/ratio_var": 7.335093687288463e-05 + }, + { + "episode": 13808, + "epoch": 0.6973737373737374, + "eps": 0, + "loss/policy_avg": 0.36910420656204224, + "loss/value_avg": 0.9228214025497437, + "lr": 2.317258114118686e-06, + "objective/entropy": -76.75859069824219, + "objective/kl": 36.92033004760742, + "objective/non_score_reward": -1.8460166454315186, + "objective/rlhf_reward": 14.872733116149902, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009935328271239996, + "policy/clipfrac_avg": 0.006773949135094881, + "policy/entropy_avg": 0.1806344985961914, + "step": 863, + "val/clipfrac_avg": 0.07717607915401459, + "val/num_eos_tokens": 514, + "val/ratio": 0.9876723289489746, + "val/ratio_var": 6.279607623582706e-05 + }, + { + "episode": 13824, + "epoch": 0.6981818181818182, + "eps": 0, + "loss/policy_avg": -0.21535149216651917, + "loss/value_avg": 0.42513978481292725, + "lr": 2.305996040265198e-06, + "objective/entropy": -77.51583862304688, + "objective/kl": 45.520545959472656, + "objective/non_score_reward": -2.276027202606201, + "objective/rlhf_reward": 14.59897232055664, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008458065567538142, + "policy/clipfrac_avg": 0.005439429543912411, + "policy/entropy_avg": 0.16998863220214844, + "step": 864, + "val/clipfrac_avg": 0.03808614984154701, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894691109657288, + "val/ratio_var": 6.017706618877128e-05 + }, + { + "episode": 13840, + "epoch": 0.6989898989898989, + "eps": 0, + "loss/policy_avg": -0.08341407775878906, + "loss/value_avg": 0.7828294038772583, + "lr": 2.29475319215209e-06, + "objective/entropy": -82.54826354980469, + "objective/kl": 39.21794891357422, + "objective/non_score_reward": -1.9608973264694214, + "objective/rlhf_reward": 14.359415054321289, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007373163243755698, + "policy/clipfrac_avg": 0.003970882389694452, + "policy/entropy_avg": 0.14882469177246094, + "step": 865, + "val/clipfrac_avg": 0.17675819993019104, + "val/num_eos_tokens": 134, + "val/ratio": 0.9905297756195068, + "val/ratio_var": 5.841355596203357e-05 + }, + { + "episode": 13856, + "epoch": 0.6997979797979798, + "eps": 0, + "loss/policy_avg": 0.029253050684928894, + "loss/value_avg": 0.8540998101234436, + "lr": 2.2835296500138664e-06, + "objective/entropy": -121.09165954589844, + "objective/kl": 34.1153564453125, + "objective/non_score_reward": -1.7057678699493408, + "objective/rlhf_reward": 13.333294868469238, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009264532127417624, + "policy/clipfrac_avg": 0.0050398847088217735, + "policy/entropy_avg": 0.18099594116210938, + "step": 866, + "val/clipfrac_avg": 0.02329857461154461, + "val/num_eos_tokens": 1353, + "val/ratio": 0.9883856177330017, + "val/ratio_var": 8.088548202067614e-05 + }, + { + "episode": 13872, + "epoch": 0.7006060606060606, + "eps": 0, + "loss/policy_avg": -0.1498204916715622, + "loss/value_avg": 0.4047302007675171, + "lr": 2.272325493947257e-06, + "objective/entropy": -49.65830993652344, + "objective/kl": 36.36531066894531, + "objective/non_score_reward": -1.8182655572891235, + "objective/rlhf_reward": 14.494234085083008, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010429773246869445, + "policy/clipfrac_avg": 0.005474097095429897, + "policy/entropy_avg": 0.18477344512939453, + "step": 867, + "val/clipfrac_avg": 0.0032526040449738503, + "val/num_eos_tokens": 101, + "val/ratio": 0.9877700805664062, + "val/ratio_var": 9.155944280792028e-05 + }, + { + "episode": 13888, + "epoch": 0.7014141414141414, + "eps": 0, + "loss/policy_avg": -0.1165279895067215, + "loss/value_avg": 0.7669879198074341, + "lr": 2.261140803910644e-06, + "objective/entropy": -77.34773254394531, + "objective/kl": 38.648643493652344, + "objective/non_score_reward": -1.9324320554733276, + "objective/rlhf_reward": 13.637880325317383, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000937658129259944, + "policy/clipfrac_avg": 0.006235727574676275, + "policy/entropy_avg": 0.19645977020263672, + "step": 868, + "val/clipfrac_avg": 0.058441728353500366, + "val/num_eos_tokens": 310, + "val/ratio": 0.9879204034805298, + "val/ratio_var": 6.414015660993755e-05 + }, + { + "episode": 13904, + "epoch": 0.7022222222222222, + "eps": 0, + "loss/policy_avg": -0.016327757388353348, + "loss/value_avg": 0.43844515085220337, + "lr": 2.2499756597234872e-06, + "objective/entropy": -163.28765869140625, + "objective/kl": 48.81157684326172, + "objective/non_score_reward": -2.4405789375305176, + "objective/rlhf_reward": 11.70395278930664, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.000844982685521245, + "policy/clipfrac_avg": 0.0032518147490918636, + "policy/entropy_avg": 0.16161727905273438, + "step": 869, + "val/clipfrac_avg": 0.014801801182329655, + "val/num_eos_tokens": 1949, + "val/ratio": 0.9885438680648804, + "val/ratio_var": 5.214377597440034e-05 + }, + { + "episode": 13920, + "epoch": 0.703030303030303, + "eps": 0, + "loss/policy_avg": -0.18964827060699463, + "loss/value_avg": 0.5136280655860901, + "lr": 2.238830141065765e-06, + "objective/entropy": -90.82677459716797, + "objective/kl": 40.3735237121582, + "objective/non_score_reward": -2.0186760425567627, + "objective/rlhf_reward": 15.153199195861816, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008435281342826784, + "policy/clipfrac_avg": 0.005789198447018862, + "policy/entropy_avg": 0.1680126190185547, + "step": 870, + "val/clipfrac_avg": 0.0648677870631218, + "val/num_eos_tokens": 555, + "val/ratio": 0.9890276193618774, + "val/ratio_var": 7.125848060240969e-05 + }, + { + "episode": 13936, + "epoch": 0.7038383838383838, + "eps": 0, + "loss/policy_avg": 0.02698252536356449, + "loss/value_avg": 0.2791609764099121, + "lr": 2.2277043274773856e-06, + "objective/entropy": -45.07421112060547, + "objective/kl": 39.21965026855469, + "objective/non_score_reward": -1.9609825611114502, + "objective/rlhf_reward": 14.624954223632812, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009133699350059032, + "policy/clipfrac_avg": 0.004030972719192505, + "policy/entropy_avg": 0.18420028686523438, + "step": 871, + "val/clipfrac_avg": 0.03648557886481285, + "val/num_eos_tokens": 232, + "val/ratio": 0.9881106615066528, + "val/ratio_var": 5.0894170271931216e-05 + }, + { + "episode": 13952, + "epoch": 0.7046464646464646, + "eps": 0, + "loss/policy_avg": -0.12599462270736694, + "loss/value_avg": 0.5007761716842651, + "lr": 2.216598298357646e-06, + "objective/entropy": -52.43424987792969, + "objective/kl": 37.084686279296875, + "objective/non_score_reward": -1.8542344570159912, + "objective/rlhf_reward": 14.18482780456543, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008421384263783693, + "policy/clipfrac_avg": 0.002664695493876934, + "policy/entropy_avg": 0.1716923713684082, + "step": 872, + "val/clipfrac_avg": 0.0010017228778451681, + "val/num_eos_tokens": 274, + "val/ratio": 0.9881088137626648, + "val/ratio_var": 6.610793934669346e-05 + }, + { + "episode": 13968, + "epoch": 0.7054545454545454, + "eps": 0, + "loss/policy_avg": 0.35178622603416443, + "loss/value_avg": 0.5846590995788574, + "lr": 2.2055121329646416e-06, + "objective/entropy": -138.92434692382812, + "objective/kl": 41.75783157348633, + "objective/non_score_reward": -2.0878915786743164, + "objective/rlhf_reward": 13.091796875, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008433774346485734, + "policy/clipfrac_avg": 0.006052501033991575, + "policy/entropy_avg": 0.16450119018554688, + "step": 873, + "val/clipfrac_avg": 0.0322103314101696, + "val/num_eos_tokens": 1144, + "val/ratio": 0.9888837337493896, + "val/ratio_var": 5.522933497559279e-05 + }, + { + "episode": 13984, + "epoch": 0.7062626262626263, + "eps": 0, + "loss/policy_avg": -0.004525661468505859, + "loss/value_avg": 0.5259770750999451, + "lr": 2.1944459104147188e-06, + "objective/entropy": -122.14696502685547, + "objective/kl": 37.81830978393555, + "objective/non_score_reward": -1.8909155130386353, + "objective/rlhf_reward": 13.296585083007812, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0006640740903094411, + "policy/clipfrac_avg": 0.004708480089902878, + "policy/entropy_avg": 0.14041614532470703, + "step": 874, + "val/clipfrac_avg": 0.10728074610233307, + "val/num_eos_tokens": 35, + "val/ratio": 0.990477979183197, + "val/ratio_var": 6.866270996397361e-05 + }, + { + "episode": 14000, + "epoch": 0.7070707070707071, + "eps": 0, + "loss/policy_avg": 0.11862924695014954, + "loss/value_avg": 1.0722770690917969, + "lr": 2.1833997096818897e-06, + "objective/entropy": -54.207332611083984, + "objective/kl": 37.62938690185547, + "objective/non_score_reward": -1.8814692497253418, + "objective/rlhf_reward": 14.2200927734375, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000859625346492976, + "policy/clipfrac_avg": 0.005390159785747528, + "policy/entropy_avg": 0.1755199432373047, + "step": 875, + "val/clipfrac_avg": 0.0593750923871994, + "val/num_eos_tokens": 201, + "val/ratio": 0.9878208637237549, + "val/ratio_var": 6.484645564341918e-05 + }, + { + "episode": 14016, + "epoch": 0.7078787878787879, + "eps": 0, + "loss/policy_avg": -0.2030085027217865, + "loss/value_avg": 0.5594953298568726, + "lr": 2.1723736095972946e-06, + "objective/entropy": -16.795242309570312, + "objective/kl": 44.697776794433594, + "objective/non_score_reward": -2.234889030456543, + "objective/rlhf_reward": 13.140110969543457, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0011914006900042295, + "policy/clipfrac_avg": 0.005575358867645264, + "policy/entropy_avg": 0.2543792724609375, + "step": 876, + "val/clipfrac_avg": 0.02393021248281002, + "val/num_eos_tokens": 24, + "val/ratio": 0.9835648536682129, + "val/ratio_var": 0.00010361165914218873 + }, + { + "episode": 14032, + "epoch": 0.7086868686868687, + "eps": 0, + "loss/policy_avg": 0.05069049820303917, + "loss/value_avg": 0.6454532146453857, + "lr": 2.161367688848618e-06, + "objective/entropy": -105.85357666015625, + "objective/kl": 50.78942108154297, + "objective/non_score_reward": -2.53947114944458, + "objective/rlhf_reward": 13.077716827392578, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008386933477595448, + "policy/clipfrac_avg": 0.003330180887132883, + "policy/entropy_avg": 0.1841726303100586, + "step": 877, + "val/clipfrac_avg": 0.010819870047271252, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886084198951721, + "val/ratio_var": 5.4145024478202686e-05 + }, + { + "episode": 14048, + "epoch": 0.7094949494949495, + "eps": 0, + "loss/policy_avg": -0.32361558079719543, + "loss/value_avg": 0.4275820851325989, + "lr": 2.1503820259795357e-06, + "objective/entropy": -193.71432495117188, + "objective/kl": 41.63684844970703, + "objective/non_score_reward": -2.0818424224853516, + "objective/rlhf_reward": 14.261907577514648, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007008577231317759, + "policy/clipfrac_avg": 0.004103646147996187, + "policy/entropy_avg": 0.1445927619934082, + "step": 878, + "val/clipfrac_avg": 7.62939453125e-06, + "val/num_eos_tokens": 323, + "val/ratio": 0.9903497695922852, + "val/ratio_var": 3.22636442433577e-05 + }, + { + "episode": 14064, + "epoch": 0.7103030303030303, + "eps": 0, + "loss/policy_avg": -0.1645442247390747, + "loss/value_avg": 0.7777354717254639, + "lr": 2.139416699389153e-06, + "objective/entropy": -45.89549255371094, + "objective/kl": 34.282958984375, + "objective/non_score_reward": -1.7141478061676025, + "objective/rlhf_reward": 14.692102432250977, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007650890620425344, + "policy/clipfrac_avg": 0.00561137031763792, + "policy/entropy_avg": 0.15566253662109375, + "step": 879, + "val/clipfrac_avg": 0.10208053886890411, + "val/num_eos_tokens": 49, + "val/ratio": 0.9895084500312805, + "val/ratio_var": 3.380152702447958e-05 + }, + { + "episode": 14080, + "epoch": 0.7111111111111111, + "eps": 0, + "loss/policy_avg": -0.023053357377648354, + "loss/value_avg": 0.521371603012085, + "lr": 2.128471787331446e-06, + "objective/entropy": -50.96327209472656, + "objective/kl": 41.30862808227539, + "objective/non_score_reward": -2.0654313564300537, + "objective/rlhf_reward": 14.450193405151367, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009377336828038096, + "policy/clipfrac_avg": 0.00421791011467576, + "policy/entropy_avg": 0.17507076263427734, + "step": 880, + "val/clipfrac_avg": 0.04448426887392998, + "val/num_eos_tokens": 368, + "val/ratio": 0.9889349937438965, + "val/ratio_var": 3.734078927664086e-05 + }, + { + "episode": 14096, + "epoch": 0.711919191919192, + "eps": 0, + "loss/policy_avg": -0.07007278501987457, + "loss/value_avg": 0.2162141352891922, + "lr": 2.117547367914705e-06, + "objective/entropy": -69.7264175415039, + "objective/kl": 41.05116653442383, + "objective/non_score_reward": -2.052558422088623, + "objective/rlhf_reward": 14.830253601074219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.001040782080963254, + "policy/clipfrac_avg": 0.0033751465380191803, + "policy/entropy_avg": 0.20317459106445312, + "step": 881, + "val/clipfrac_avg": 0.03193364292383194, + "val/num_eos_tokens": 551, + "val/ratio": 0.9861003160476685, + "val/ratio_var": 8.361194340977818e-05 + }, + { + "episode": 14112, + "epoch": 0.7127272727272728, + "eps": 0, + "loss/policy_avg": -0.07847104221582413, + "loss/value_avg": 0.2657424807548523, + "lr": 2.1066435191009717e-06, + "objective/entropy": -86.86426544189453, + "objective/kl": 45.00962829589844, + "objective/non_score_reward": -2.250481367111206, + "objective/rlhf_reward": 13.913580894470215, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010606081923469901, + "policy/clipfrac_avg": 0.0069802612997591496, + "policy/entropy_avg": 0.20689773559570312, + "step": 882, + "val/clipfrac_avg": 0.038475796580314636, + "val/num_eos_tokens": 546, + "val/ratio": 0.985522449016571, + "val/ratio_var": 8.580441499361768e-05 + }, + { + "episode": 14128, + "epoch": 0.7135353535353536, + "eps": 0, + "loss/policy_avg": -0.1323227733373642, + "loss/value_avg": 0.21754738688468933, + "lr": 2.095760318705487e-06, + "objective/entropy": -109.19290161132812, + "objective/kl": 42.61073303222656, + "objective/non_score_reward": -2.1305365562438965, + "objective/rlhf_reward": 14.447587966918945, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010359378065913916, + "policy/clipfrac_avg": 0.006892787292599678, + "policy/entropy_avg": 0.20017433166503906, + "step": 883, + "val/clipfrac_avg": 0.010656789876520634, + "val/num_eos_tokens": 36, + "val/ratio": 0.9864689111709595, + "val/ratio_var": 4.844724026042968e-05 + }, + { + "episode": 14144, + "epoch": 0.7143434343434344, + "eps": 0, + "loss/policy_avg": -0.1507091522216797, + "loss/value_avg": 0.2665415108203888, + "lr": 2.0848978443961314e-06, + "objective/entropy": -119.52471923828125, + "objective/kl": 43.517696380615234, + "objective/non_score_reward": -2.17588472366333, + "objective/rlhf_reward": 14.300678253173828, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000938809709623456, + "policy/clipfrac_avg": 0.012131594121456146, + "policy/entropy_avg": 0.18040943145751953, + "step": 884, + "val/clipfrac_avg": 0.006075765937566757, + "val/num_eos_tokens": 620, + "val/ratio": 0.9873965978622437, + "val/ratio_var": 6.621635111514479e-05 + }, + { + "episode": 14160, + "epoch": 0.7151515151515152, + "eps": 0, + "loss/policy_avg": -0.028530046343803406, + "loss/value_avg": 0.434810072183609, + "lr": 2.074056173692881e-06, + "objective/entropy": -62.273380279541016, + "objective/kl": 45.77116394042969, + "objective/non_score_reward": -2.2885584831237793, + "objective/rlhf_reward": 14.617691040039062, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009718666551634669, + "policy/clipfrac_avg": 0.004924894310534, + "policy/entropy_avg": 0.20779800415039062, + "step": 885, + "val/clipfrac_avg": 0.036309417337179184, + "val/num_eos_tokens": 179, + "val/ratio": 0.9870821237564087, + "val/ratio_var": 8.347785478690639e-05 + }, + { + "episode": 14176, + "epoch": 0.7159595959595959, + "eps": 0, + "loss/policy_avg": -0.15033547580242157, + "loss/value_avg": 0.6043248772621155, + "lr": 2.0632353839672415e-06, + "objective/entropy": -84.31056213378906, + "objective/kl": 32.41828918457031, + "objective/non_score_reward": -1.6209145784378052, + "objective/rlhf_reward": 15.332210540771484, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008589382632635534, + "policy/clipfrac_avg": 0.002752305706962943, + "policy/entropy_avg": 0.18497180938720703, + "step": 886, + "val/clipfrac_avg": 0.03947104513645172, + "val/num_eos_tokens": 180, + "val/ratio": 0.9879313707351685, + "val/ratio_var": 5.601351585937664e-05 + }, + { + "episode": 14192, + "epoch": 0.7167676767676767, + "eps": 0, + "loss/policy_avg": 0.2279771864414215, + "loss/value_avg": 1.2074341773986816, + "lr": 2.0524355524417017e-06, + "objective/entropy": -51.082984924316406, + "objective/kl": 36.84368133544922, + "objective/non_score_reward": -1.842184066772461, + "objective/rlhf_reward": 14.087503433227539, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010461971396580338, + "policy/clipfrac_avg": 0.004587469156831503, + "policy/entropy_avg": 0.22597980499267578, + "step": 887, + "val/clipfrac_avg": 0.015519710257649422, + "val/num_eos_tokens": 448, + "val/ratio": 0.9849185943603516, + "val/ratio_var": 6.34926327620633e-05 + }, + { + "episode": 14208, + "epoch": 0.7175757575757575, + "eps": 0, + "loss/policy_avg": -0.026267291978001595, + "loss/value_avg": 0.5965927839279175, + "lr": 2.041656756189184e-06, + "objective/entropy": -154.47012329101562, + "objective/kl": 42.720699310302734, + "objective/non_score_reward": -2.1360349655151367, + "objective/rlhf_reward": 14.957715034484863, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009299394441768527, + "policy/clipfrac_avg": 0.008321288973093033, + "policy/entropy_avg": 0.1751893162727356, + "step": 888, + "val/clipfrac_avg": 0.01875000074505806, + "val/num_eos_tokens": 428, + "val/ratio": 0.9883604049682617, + "val/ratio_var": 7.325677870539948e-05 + }, + { + "episode": 14224, + "epoch": 0.7183838383838383, + "eps": 0, + "loss/policy_avg": 0.40078702569007874, + "loss/value_avg": 1.1534854173660278, + "lr": 2.030899072132493e-06, + "objective/entropy": -128.48831176757812, + "objective/kl": 33.898155212402344, + "objective/non_score_reward": -1.6949079036712646, + "objective/rlhf_reward": 12.598060607910156, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006705562118440866, + "policy/clipfrac_avg": 0.005831941030919552, + "policy/entropy_avg": 0.21662330627441406, + "step": 889, + "val/clipfrac_avg": 0.02483724057674408, + "val/num_eos_tokens": 47, + "val/ratio": 0.9912885427474976, + "val/ratio_var": 7.058407936710864e-05 + }, + { + "episode": 14240, + "epoch": 0.7191919191919192, + "eps": 0, + "loss/policy_avg": -0.0855930745601654, + "loss/value_avg": 0.32921552658081055, + "lr": 2.0201625770437647e-06, + "objective/entropy": -86.9806137084961, + "objective/kl": 40.06652069091797, + "objective/non_score_reward": -2.003325939178467, + "objective/rlhf_reward": 14.035737037658691, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008815613109618425, + "policy/clipfrac_avg": 0.006017718929797411, + "policy/entropy_avg": 0.1755352020263672, + "step": 890, + "val/clipfrac_avg": 0.06256753951311111, + "val/num_eos_tokens": 20, + "val/ratio": 0.9872984290122986, + "val/ratio_var": 8.527067984687164e-05 + }, + { + "episode": 14256, + "epoch": 0.72, + "eps": 0, + "loss/policy_avg": -0.17781905829906464, + "loss/value_avg": 0.2275725156068802, + "lr": 2.00944734754392e-06, + "objective/entropy": -99.16207122802734, + "objective/kl": 41.95650863647461, + "objective/non_score_reward": -2.097825288772583, + "objective/rlhf_reward": 13.74592399597168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007897773757576942, + "policy/clipfrac_avg": 0.0043031517416238785, + "policy/entropy_avg": 0.1585559844970703, + "step": 891, + "val/clipfrac_avg": 0.012334126979112625, + "val/num_eos_tokens": 424, + "val/ratio": 0.9894897937774658, + "val/ratio_var": 5.430886085378006e-05 + }, + { + "episode": 14272, + "epoch": 0.7208080808080808, + "eps": 0, + "loss/policy_avg": -0.36396488547325134, + "loss/value_avg": 0.8662234544754028, + "lr": 1.9987534601021225e-06, + "objective/entropy": -61.42802047729492, + "objective/kl": 37.07469177246094, + "objective/non_score_reward": -1.8537347316741943, + "objective/rlhf_reward": 13.044702529907227, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008486385922878981, + "policy/clipfrac_avg": 0.006757801398634911, + "policy/entropy_avg": 0.1866321563720703, + "step": 892, + "val/clipfrac_avg": 0.054801926016807556, + "val/num_eos_tokens": 584, + "val/ratio": 0.9883260726928711, + "val/ratio_var": 6.7303488322068e-05 + }, + { + "episode": 14288, + "epoch": 0.7216161616161616, + "eps": 0, + "loss/policy_avg": -0.12171509861946106, + "loss/value_avg": 0.3499939441680908, + "lr": 1.988080991035223e-06, + "objective/entropy": -58.38874053955078, + "objective/kl": 42.03372573852539, + "objective/non_score_reward": -2.1016862392425537, + "objective/rlhf_reward": 12.828001022338867, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008078413084149361, + "policy/clipfrac_avg": 0.004710954613983631, + "policy/entropy_avg": 0.16219139099121094, + "step": 893, + "val/clipfrac_avg": 0.03385913744568825, + "val/num_eos_tokens": 211, + "val/ratio": 0.9909918904304504, + "val/ratio_var": 4.974031253368594e-05 + }, + { + "episode": 14304, + "epoch": 0.7224242424242424, + "eps": 0, + "loss/policy_avg": 0.15615974366664886, + "loss/value_avg": 0.4985978901386261, + "lr": 1.977430016507222e-06, + "objective/entropy": -72.03929138183594, + "objective/kl": 43.86260986328125, + "objective/non_score_reward": -2.1931304931640625, + "objective/rlhf_reward": 13.322494506835938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009859772399067879, + "policy/clipfrac_avg": 0.004861406981945038, + "policy/entropy_avg": 0.19806480407714844, + "step": 894, + "val/clipfrac_avg": 0.008869262412190437, + "val/num_eos_tokens": 389, + "val/ratio": 0.9869673252105713, + "val/ratio_var": 5.9525649703573436e-05 + }, + { + "episode": 14320, + "epoch": 0.7232323232323232, + "eps": 0, + "loss/policy_avg": -0.018653089180588722, + "loss/value_avg": 0.3236011266708374, + "lr": 1.966800612528723e-06, + "objective/entropy": -30.8804874420166, + "objective/kl": 38.795677185058594, + "objective/non_score_reward": -1.9397838115692139, + "objective/rlhf_reward": 12.958653450012207, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011067797895520926, + "policy/clipfrac_avg": 0.004753993824124336, + "policy/entropy_avg": 0.21401214599609375, + "step": 895, + "val/clipfrac_avg": 0.013285137712955475, + "val/num_eos_tokens": 190, + "val/ratio": 0.9844212532043457, + "val/ratio_var": 5.578770287684165e-05 + }, + { + "episode": 14336, + "epoch": 0.724040404040404, + "eps": 0, + "loss/policy_avg": -0.04005177319049835, + "loss/value_avg": 0.564488410949707, + "lr": 1.956192854956397e-06, + "objective/entropy": -88.01570129394531, + "objective/kl": 40.9593505859375, + "objective/non_score_reward": -2.0479674339294434, + "objective/rlhf_reward": 13.350469589233398, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010309319477528334, + "policy/clipfrac_avg": 0.005483411252498627, + "policy/entropy_avg": 0.21037864685058594, + "step": 896, + "val/clipfrac_avg": 0.01046415138989687, + "val/num_eos_tokens": 539, + "val/ratio": 0.9865807890892029, + "val/ratio_var": 9.776368096936494e-05 + }, + { + "episode": 14352, + "epoch": 0.7248484848484849, + "eps": 0, + "loss/policy_avg": -0.1358407437801361, + "loss/value_avg": 0.536889910697937, + "lr": 1.945606819492429e-06, + "objective/entropy": -98.2094955444336, + "objective/kl": 37.47423553466797, + "objective/non_score_reward": -1.8737118244171143, + "objective/rlhf_reward": 13.634100914001465, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007985340780578554, + "policy/clipfrac_avg": 0.005063176155090332, + "policy/entropy_avg": 0.14582359790802002, + "step": 897, + "val/clipfrac_avg": 0.06062069162726402, + "val/num_eos_tokens": 1182, + "val/ratio": 0.9903557300567627, + "val/ratio_var": 5.06751166540198e-05 + }, + { + "episode": 14368, + "epoch": 0.7256565656565657, + "eps": 0, + "loss/policy_avg": 0.011427253484725952, + "loss/value_avg": 0.263162761926651, + "lr": 1.9350425816839875e-06, + "objective/entropy": -110.30941772460938, + "objective/kl": 41.13695526123047, + "objective/non_score_reward": -2.0568478107452393, + "objective/rlhf_reward": 13.72440242767334, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011626393534243107, + "policy/clipfrac_avg": 0.006754214409738779, + "policy/entropy_avg": 0.2226581573486328, + "step": 898, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 10, + "val/ratio": 0.9850178956985474, + "val/ratio_var": 8.876781794242561e-05 + }, + { + "episode": 14384, + "epoch": 0.7264646464646465, + "eps": 0, + "loss/policy_avg": 0.11541193723678589, + "loss/value_avg": 0.4631091356277466, + "lr": 1.9245002169226814e-06, + "objective/entropy": -101.70489501953125, + "objective/kl": 48.557960510253906, + "objective/non_score_reward": -2.4278979301452637, + "objective/rlhf_reward": 13.314289093017578, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009826980531215668, + "policy/clipfrac_avg": 0.004610124975442886, + "policy/entropy_avg": 0.1724681854248047, + "step": 899, + "val/clipfrac_avg": 0.013620924204587936, + "val/num_eos_tokens": 516, + "val/ratio": 0.9876973628997803, + "val/ratio_var": 5.4934793297434226e-05 + }, + { + "episode": 14400, + "epoch": 0.7272727272727273, + "eps": 0, + "loss/policy_avg": 0.07936800271272659, + "loss/value_avg": 0.6616516709327698, + "lr": 1.913979800444021e-06, + "objective/entropy": -91.96019744873047, + "objective/kl": 41.26939010620117, + "objective/non_score_reward": -2.063469409942627, + "objective/rlhf_reward": 12.522467613220215, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0009332017507404089, + "policy/clipfrac_avg": 0.0043076397851109505, + "policy/entropy_avg": 0.18123245239257812, + "step": 900, + "val/clipfrac_avg": 0.01476163137704134, + "val/num_eos_tokens": 595, + "val/ratio": 0.9875973463058472, + "val/ratio_var": 7.475823076674715e-05 + }, + { + "episode": 14416, + "epoch": 0.7280808080808081, + "eps": 0, + "loss/policy_avg": -0.02671997621655464, + "loss/value_avg": 0.7174835205078125, + "lr": 1.9034814073268898e-06, + "objective/entropy": -59.236602783203125, + "objective/kl": 39.73054122924805, + "objective/non_score_reward": -1.9865270853042603, + "objective/rlhf_reward": 12.435348510742188, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011704354546964169, + "policy/clipfrac_avg": 0.004937891382724047, + "policy/entropy_avg": 0.22887516021728516, + "step": 901, + "val/clipfrac_avg": 0.13261640071868896, + "val/num_eos_tokens": 121, + "val/ratio": 0.9848372936248779, + "val/ratio_var": 8.883672853698954e-05 + }, + { + "episode": 14432, + "epoch": 0.7288888888888889, + "eps": 0, + "loss/policy_avg": 0.05788304656744003, + "loss/value_avg": 0.2591496407985687, + "lr": 1.8930051124929904e-06, + "objective/entropy": -53.46650695800781, + "objective/kl": 46.268035888671875, + "objective/non_score_reward": -2.313401937484741, + "objective/rlhf_reward": 14.38972282409668, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010022292844951153, + "policy/clipfrac_avg": 0.004066841676831245, + "policy/entropy_avg": 0.20316123962402344, + "step": 902, + "val/clipfrac_avg": 0.013965151272714138, + "val/num_eos_tokens": 455, + "val/ratio": 0.9866280555725098, + "val/ratio_var": 5.229823727859184e-05 + }, + { + "episode": 14448, + "epoch": 0.7296969696969697, + "eps": 0, + "loss/policy_avg": -0.11720985174179077, + "loss/value_avg": 0.23323997855186462, + "lr": 1.8825509907063328e-06, + "objective/entropy": -65.6883773803711, + "objective/kl": 46.209991455078125, + "objective/non_score_reward": -2.310499668121338, + "objective/rlhf_reward": 16.78325080871582, + "objective/scores": 19.125, + "policy/approxkl_avg": 0.0010066109243780375, + "policy/clipfrac_avg": 0.005406823009252548, + "policy/entropy_avg": 0.20101547241210938, + "step": 903, + "val/clipfrac_avg": 0.015051587484776974, + "val/num_eos_tokens": 205, + "val/ratio": 0.9865299463272095, + "val/ratio_var": 5.01565846207086e-05 + }, + { + "episode": 14464, + "epoch": 0.7305050505050505, + "eps": 0, + "loss/policy_avg": -0.4681462049484253, + "loss/value_avg": 0.8052223920822144, + "lr": 1.87211911657268e-06, + "objective/entropy": -118.0733642578125, + "objective/kl": 37.29925537109375, + "objective/non_score_reward": -1.8649628162384033, + "objective/rlhf_reward": 15.885037422180176, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0007588897133246064, + "policy/clipfrac_avg": 0.0023906652349978685, + "policy/entropy_avg": 0.15973949432373047, + "step": 904, + "val/clipfrac_avg": 0.13810205459594727, + "val/num_eos_tokens": 155, + "val/ratio": 0.9902135729789734, + "val/ratio_var": 7.25651771062985e-05 + }, + { + "episode": 14480, + "epoch": 0.7313131313131314, + "eps": 0, + "loss/policy_avg": 0.19160020351409912, + "loss/value_avg": 0.9071890115737915, + "lr": 1.8617095645390332e-06, + "objective/entropy": -22.496440887451172, + "objective/kl": 39.160484313964844, + "objective/non_score_reward": -1.958024263381958, + "objective/rlhf_reward": 12.323225975036621, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0010718535631895065, + "policy/clipfrac_avg": 0.004276500083506107, + "policy/entropy_avg": 0.19223785400390625, + "step": 905, + "val/clipfrac_avg": 0.03053325042128563, + "val/num_eos_tokens": 240, + "val/ratio": 0.9871090650558472, + "val/ratio_var": 4.5721961214439943e-05 + }, + { + "episode": 14496, + "epoch": 0.7321212121212122, + "eps": 0, + "loss/policy_avg": -0.11877916008234024, + "loss/value_avg": 0.6873422265052795, + "lr": 1.8513224088930814e-06, + "objective/entropy": -52.41043472290039, + "objective/kl": 30.631162643432617, + "objective/non_score_reward": -1.5315582752227783, + "objective/rlhf_reward": 16.296566009521484, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.001107763615436852, + "policy/clipfrac_avg": 0.006524171680212021, + "policy/entropy_avg": 0.2064342498779297, + "step": 906, + "val/clipfrac_avg": 0.000294811325147748, + "val/num_eos_tokens": 261, + "val/ratio": 0.9861389398574829, + "val/ratio_var": 6.195842433953658e-05 + }, + { + "episode": 14512, + "epoch": 0.7329292929292929, + "eps": 0, + "loss/policy_avg": 0.10565789043903351, + "loss/value_avg": 0.5067005157470703, + "lr": 1.8409577237626935e-06, + "objective/entropy": -61.856544494628906, + "objective/kl": 43.33234405517578, + "objective/non_score_reward": -2.1666173934936523, + "objective/rlhf_reward": 13.145882606506348, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010527942795306444, + "policy/clipfrac_avg": 0.004575061611831188, + "policy/entropy_avg": 0.18899154663085938, + "step": 907, + "val/clipfrac_avg": 0.005580357275903225, + "val/num_eos_tokens": 496, + "val/ratio": 0.9872788190841675, + "val/ratio_var": 7.066375110298395e-05 + }, + { + "episode": 14528, + "epoch": 0.7337373737373737, + "eps": 0, + "loss/policy_avg": 0.04191252216696739, + "loss/value_avg": 0.30707108974456787, + "lr": 1.8306155831153699e-06, + "objective/entropy": -91.67915344238281, + "objective/kl": 37.619895935058594, + "objective/non_score_reward": -1.8809947967529297, + "objective/rlhf_reward": 14.22056770324707, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009494669502601027, + "policy/clipfrac_avg": 0.007099117152392864, + "policy/entropy_avg": 0.17319107055664062, + "step": 908, + "val/clipfrac_avg": 0.07242873311042786, + "val/num_eos_tokens": 437, + "val/ratio": 0.9883796572685242, + "val/ratio_var": 7.36714864615351e-05 + }, + { + "episode": 14544, + "epoch": 0.7345454545454545, + "eps": 0, + "loss/policy_avg": 0.15317559242248535, + "loss/value_avg": 0.7379442453384399, + "lr": 1.8202960607577246e-06, + "objective/entropy": -92.03844451904297, + "objective/kl": 36.91444396972656, + "objective/non_score_reward": -1.8457221984863281, + "objective/rlhf_reward": 13.802715301513672, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0009770962642505765, + "policy/clipfrac_avg": 0.005569060333073139, + "policy/entropy_avg": 0.18958091735839844, + "step": 909, + "val/clipfrac_avg": 0.04051949828863144, + "val/num_eos_tokens": 56, + "val/ratio": 0.988923192024231, + "val/ratio_var": 6.142368511063978e-05 + }, + { + "episode": 14560, + "epoch": 0.7353535353535353, + "eps": 0, + "loss/policy_avg": 0.1602431833744049, + "loss/value_avg": 0.3025469183921814, + "lr": 1.809999230334958e-06, + "objective/entropy": -111.80148315429688, + "objective/kl": 38.83802032470703, + "objective/non_score_reward": -1.9419012069702148, + "objective/rlhf_reward": 14.401848793029785, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000904000480659306, + "policy/clipfrac_avg": 0.005316127557307482, + "policy/entropy_avg": 0.17273902893066406, + "step": 910, + "val/clipfrac_avg": 0.00018382353300694376, + "val/num_eos_tokens": 342, + "val/ratio": 0.9891130924224854, + "val/ratio_var": 7.234779332065955e-05 + }, + { + "episode": 14576, + "epoch": 0.7361616161616161, + "eps": 0, + "loss/policy_avg": 0.09882547706365585, + "loss/value_avg": 0.6121327877044678, + "lr": 1.7997251653303249e-06, + "objective/entropy": -98.02986145019531, + "objective/kl": 44.55670166015625, + "objective/non_score_reward": -2.227834939956665, + "objective/rlhf_reward": 13.092477798461914, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009336632210761309, + "policy/clipfrac_avg": 0.004822542425245047, + "policy/entropy_avg": 0.17920780181884766, + "step": 911, + "val/clipfrac_avg": 0.0050772735849022865, + "val/num_eos_tokens": 426, + "val/ratio": 0.9875223636627197, + "val/ratio_var": 6.380004924722016e-05 + }, + { + "episode": 14592, + "epoch": 0.7369696969696969, + "eps": 0, + "loss/policy_avg": -0.15951846539974213, + "loss/value_avg": 0.5648845434188843, + "lr": 1.7894739390646227e-06, + "objective/entropy": -57.16059875488281, + "objective/kl": 36.05466079711914, + "objective/non_score_reward": -1.802733063697815, + "objective/rlhf_reward": 13.119142532348633, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009670511353760958, + "policy/clipfrac_avg": 0.00586615689098835, + "policy/entropy_avg": 0.1721019744873047, + "step": 912, + "val/clipfrac_avg": 0.0008223684271797538, + "val/num_eos_tokens": 45, + "val/ratio": 0.9892112016677856, + "val/ratio_var": 4.9591733841225505e-05 + }, + { + "episode": 14608, + "epoch": 0.7377777777777778, + "eps": 0, + "loss/policy_avg": 0.1940194070339203, + "loss/value_avg": 0.3815060555934906, + "lr": 1.7792456246956535e-06, + "objective/entropy": -146.6658935546875, + "objective/kl": 46.18011474609375, + "objective/non_score_reward": -2.3090057373046875, + "objective/rlhf_reward": 14.511306762695312, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009307245491072536, + "policy/clipfrac_avg": 0.006143895909190178, + "policy/entropy_avg": 0.18556424975395203, + "step": 913, + "val/clipfrac_avg": 0.0180903859436512, + "val/num_eos_tokens": 698, + "val/ratio": 0.9874189496040344, + "val/ratio_var": 0.00011522904242156073 + }, + { + "episode": 14624, + "epoch": 0.7385858585858586, + "eps": 0, + "loss/policy_avg": -0.02095976285636425, + "loss/value_avg": 0.7224372625350952, + "lr": 1.7690402952177117e-06, + "objective/entropy": -84.84890747070312, + "objective/kl": 35.78252410888672, + "objective/non_score_reward": -1.7891263961791992, + "objective/rlhf_reward": 13.2655611038208, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009321504039689898, + "policy/clipfrac_avg": 0.005007730796933174, + "policy/entropy_avg": 0.18702125549316406, + "step": 914, + "val/clipfrac_avg": 0.023768942803144455, + "val/num_eos_tokens": 403, + "val/ratio": 0.9874284267425537, + "val/ratio_var": 8.100390550680459e-05 + }, + { + "episode": 14640, + "epoch": 0.7393939393939394, + "eps": 0, + "loss/policy_avg": -0.3134731650352478, + "loss/value_avg": 0.5765621066093445, + "lr": 1.7588580234610592e-06, + "objective/entropy": -63.22113037109375, + "objective/kl": 38.5015869140625, + "objective/non_score_reward": -1.925079345703125, + "objective/rlhf_reward": 13.918670654296875, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008501409320160747, + "policy/clipfrac_avg": 0.004706401377916336, + "policy/entropy_avg": 0.1600513458251953, + "step": 915, + "val/clipfrac_avg": 0.06931675970554352, + "val/num_eos_tokens": 171, + "val/ratio": 0.9882992506027222, + "val/ratio_var": 6.202542135724798e-05 + }, + { + "episode": 14656, + "epoch": 0.7402020202020202, + "eps": 0, + "loss/policy_avg": -0.2850199043750763, + "loss/value_avg": 0.8679437637329102, + "lr": 1.7486988820914063e-06, + "objective/entropy": -43.271400451660156, + "objective/kl": 42.955291748046875, + "objective/non_score_reward": -2.1477646827697754, + "objective/rlhf_reward": 13.875673294067383, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009495788253843784, + "policy/clipfrac_avg": 0.005583056248724461, + "policy/entropy_avg": 0.17831039428710938, + "step": 916, + "val/clipfrac_avg": 0.06379853188991547, + "val/num_eos_tokens": 103, + "val/ratio": 0.9876798391342163, + "val/ratio_var": 4.0028760849963874e-05 + }, + { + "episode": 14672, + "epoch": 0.741010101010101, + "eps": 0, + "loss/policy_avg": 0.21622401475906372, + "loss/value_avg": 0.22584834694862366, + "lr": 1.7385629436093958e-06, + "objective/entropy": -79.67193603515625, + "objective/kl": 44.639923095703125, + "objective/non_score_reward": -2.2319962978363037, + "objective/rlhf_reward": 13.463315963745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008941494161263108, + "policy/clipfrac_avg": 0.006674608215689659, + "policy/entropy_avg": 0.1719951629638672, + "step": 917, + "val/clipfrac_avg": 0.029584797099232674, + "val/num_eos_tokens": 50, + "val/ratio": 0.9886019825935364, + "val/ratio_var": 4.6610963181592524e-05 + }, + { + "episode": 14688, + "epoch": 0.7418181818181818, + "eps": 0, + "loss/policy_avg": -0.0006486065685749054, + "loss/value_avg": 0.2962718605995178, + "lr": 1.728450280350079e-06, + "objective/entropy": -68.00552368164062, + "objective/kl": 38.6593132019043, + "objective/non_score_reward": -1.9329655170440674, + "objective/rlhf_reward": 13.887346267700195, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008251155959442258, + "policy/clipfrac_avg": 0.005509347654879093, + "policy/entropy_avg": 0.1643085479736328, + "step": 918, + "val/clipfrac_avg": 0.003826581174507737, + "val/num_eos_tokens": 44, + "val/ratio": 0.9888970851898193, + "val/ratio_var": 3.0508910640492104e-05 + }, + { + "episode": 14704, + "epoch": 0.7426262626262626, + "eps": 0, + "loss/policy_avg": -0.3462671935558319, + "loss/value_avg": 0.7152901887893677, + "lr": 1.7183609644824096e-06, + "objective/entropy": -93.26936340332031, + "objective/kl": 40.42371368408203, + "objective/non_score_reward": -2.021185874938965, + "objective/rlhf_reward": 15.166314125061035, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009743100963532925, + "policy/clipfrac_avg": 0.0028937258757650852, + "policy/entropy_avg": 0.18361568450927734, + "step": 919, + "val/clipfrac_avg": 0.049027521163225174, + "val/num_eos_tokens": 136, + "val/ratio": 0.9877507090568542, + "val/ratio_var": 5.374600732466206e-05 + }, + { + "episode": 14720, + "epoch": 0.7434343434343434, + "eps": 0, + "loss/policy_avg": 0.1274300217628479, + "loss/value_avg": 1.230905294418335, + "lr": 1.708295068008718e-06, + "objective/entropy": -81.48149108886719, + "objective/kl": 36.17670440673828, + "objective/non_score_reward": -1.8088352680206299, + "objective/rlhf_reward": 12.589601516723633, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008495738729834557, + "policy/clipfrac_avg": 0.0038070818409323692, + "policy/entropy_avg": 0.16942787170410156, + "step": 920, + "val/clipfrac_avg": 0.007427365053445101, + "val/num_eos_tokens": 270, + "val/ratio": 0.9902092218399048, + "val/ratio_var": 5.2669711294583976e-05 + }, + { + "episode": 14736, + "epoch": 0.7442424242424243, + "eps": 0, + "loss/policy_avg": -0.257866770029068, + "loss/value_avg": 0.5942734479904175, + "lr": 1.6982526627642043e-06, + "objective/entropy": -74.04690551757812, + "objective/kl": 42.419307708740234, + "objective/non_score_reward": -2.1209654808044434, + "objective/rlhf_reward": 13.511846542358398, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000907333567738533, + "policy/clipfrac_avg": 0.004857117310166359, + "policy/entropy_avg": 0.18361282348632812, + "step": 921, + "val/clipfrac_avg": 0.043217435479164124, + "val/num_eos_tokens": 56, + "val/ratio": 0.9877738356590271, + "val/ratio_var": 6.433074304368347e-05 + }, + { + "episode": 14752, + "epoch": 0.7450505050505051, + "eps": 0, + "loss/policy_avg": 0.12189102917909622, + "loss/value_avg": 0.41360408067703247, + "lr": 1.6882338204164212e-06, + "objective/entropy": -54.975791931152344, + "objective/kl": 35.72700881958008, + "objective/non_score_reward": -1.7863504886627197, + "objective/rlhf_reward": 13.39333724975586, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010183433769270778, + "policy/clipfrac_avg": 0.003768398892134428, + "policy/entropy_avg": 0.1844940185546875, + "step": 922, + "val/clipfrac_avg": 0.0008610248914919794, + "val/num_eos_tokens": 362, + "val/ratio": 0.9872738122940063, + "val/ratio_var": 3.0919560231268406e-05 + }, + { + "episode": 14768, + "epoch": 0.7458585858585859, + "eps": 0, + "loss/policy_avg": -0.07059650868177414, + "loss/value_avg": 0.6463966369628906, + "lr": 1.67823861246477e-06, + "objective/entropy": -15.746962547302246, + "objective/kl": 40.59729766845703, + "objective/non_score_reward": -2.029865026473999, + "objective/rlhf_reward": 12.388103485107422, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010770086664706469, + "policy/clipfrac_avg": 0.005413630045950413, + "policy/entropy_avg": 0.21207809448242188, + "step": 923, + "val/clipfrac_avg": 0.00856552179902792, + "val/num_eos_tokens": 175, + "val/ratio": 0.9861268401145935, + "val/ratio_var": 5.63512985536363e-05 + }, + { + "episode": 14784, + "epoch": 0.7466666666666667, + "eps": 0, + "loss/policy_avg": 0.06915351748466492, + "loss/value_avg": 1.4651566743850708, + "lr": 1.6682671102399806e-06, + "objective/entropy": -118.91422271728516, + "objective/kl": 34.25519943237305, + "objective/non_score_reward": -1.7127599716186523, + "objective/rlhf_reward": 12.224739074707031, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0007098158821463585, + "policy/clipfrac_avg": 0.0022262982092797756, + "policy/entropy_avg": 0.1386256217956543, + "step": 924, + "val/clipfrac_avg": 0.06050518900156021, + "val/num_eos_tokens": 270, + "val/ratio": 0.9912421703338623, + "val/ratio_var": 4.530951628112234e-05 + }, + { + "episode": 14800, + "epoch": 0.7474747474747475, + "eps": 0, + "loss/policy_avg": 0.18248242139816284, + "loss/value_avg": 1.6025516986846924, + "lr": 1.6583193849036095e-06, + "objective/entropy": -81.5009765625, + "objective/kl": 38.95854949951172, + "objective/non_score_reward": -1.947927474975586, + "objective/rlhf_reward": 15.450510025024414, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009277502540498972, + "policy/clipfrac_avg": 0.004952313844114542, + "policy/entropy_avg": 0.16534215211868286, + "step": 925, + "val/clipfrac_avg": 0.016582708805799484, + "val/num_eos_tokens": 323, + "val/ratio": 0.9883300065994263, + "val/ratio_var": 5.890153624932282e-05 + }, + { + "episode": 14816, + "epoch": 0.7482828282828283, + "eps": 0, + "loss/policy_avg": -0.16795751452445984, + "loss/value_avg": 0.3992899954319, + "lr": 1.6483955074475278e-06, + "objective/entropy": -46.540870666503906, + "objective/kl": 35.18474578857422, + "objective/non_score_reward": -1.75923752784729, + "objective/rlhf_reward": 13.225137710571289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009727986180223525, + "policy/clipfrac_avg": 0.0023795529268682003, + "policy/entropy_avg": 0.21690034866333008, + "step": 926, + "val/clipfrac_avg": 0.0444885715842247, + "val/num_eos_tokens": 163, + "val/ratio": 0.9869061708450317, + "val/ratio_var": 6.481046148110181e-05 + }, + { + "episode": 14832, + "epoch": 0.7490909090909091, + "eps": 0, + "loss/policy_avg": -0.3186694383621216, + "loss/value_avg": 0.933090090751648, + "lr": 1.6384955486934157e-06, + "objective/entropy": -73.27378845214844, + "objective/kl": 33.047142028808594, + "objective/non_score_reward": -1.6523573398590088, + "objective/rlhf_reward": 13.33201789855957, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008205980993807316, + "policy/clipfrac_avg": 0.0039194016717374325, + "policy/entropy_avg": 0.17404580116271973, + "step": 927, + "val/clipfrac_avg": 0.07463380694389343, + "val/num_eos_tokens": 165, + "val/ratio": 0.9890973567962646, + "val/ratio_var": 8.332987636094913e-05 + }, + { + "episode": 14848, + "epoch": 0.74989898989899, + "eps": 0, + "loss/policy_avg": -0.018651030957698822, + "loss/value_avg": 0.23730480670928955, + "lr": 1.6286195792922655e-06, + "objective/entropy": -108.95191192626953, + "objective/kl": 41.2000846862793, + "objective/non_score_reward": -2.060004472732544, + "objective/rlhf_reward": 14.135308265686035, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007390918326564133, + "policy/clipfrac_avg": 0.0021800659596920013, + "policy/entropy_avg": 0.15365886688232422, + "step": 928, + "val/clipfrac_avg": 0.02866453304886818, + "val/num_eos_tokens": 497, + "val/ratio": 0.989920973777771, + "val/ratio_var": 5.118203625897877e-05 + }, + { + "episode": 14864, + "epoch": 0.7507070707070707, + "eps": 0, + "loss/policy_avg": 0.11356082558631897, + "loss/value_avg": 0.5069226622581482, + "lr": 1.6187676697238564e-06, + "objective/entropy": -53.78815841674805, + "objective/kl": 47.88815689086914, + "objective/non_score_reward": -2.3944079875946045, + "objective/rlhf_reward": 12.574342727661133, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008938192040659487, + "policy/clipfrac_avg": 0.003149396274238825, + "policy/entropy_avg": 0.181840181350708, + "step": 929, + "val/clipfrac_avg": 0.0172991082072258, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889012575149536, + "val/ratio_var": 3.912192551069893e-05 + }, + { + "episode": 14880, + "epoch": 0.7515151515151515, + "eps": 0, + "loss/policy_avg": -0.07865975797176361, + "loss/value_avg": 0.41433531045913696, + "lr": 1.6089398902962767e-06, + "objective/entropy": -104.1807861328125, + "objective/kl": 37.49846649169922, + "objective/non_score_reward": -1.8749234676361084, + "objective/rlhf_reward": 15.953201293945312, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009115774882957339, + "policy/clipfrac_avg": 0.005451949778944254, + "policy/entropy_avg": 0.1745290756225586, + "step": 930, + "val/clipfrac_avg": 0.058040205389261246, + "val/num_eos_tokens": 27, + "val/ratio": 0.9887331128120422, + "val/ratio_var": 6.705939449602738e-05 + }, + { + "episode": 14896, + "epoch": 0.7523232323232323, + "eps": 0, + "loss/policy_avg": -0.035402484238147736, + "loss/value_avg": 0.5358971357345581, + "lr": 1.5991363111454023e-06, + "objective/entropy": -97.50758361816406, + "objective/kl": 42.68102264404297, + "objective/non_score_reward": -2.1340510845184326, + "objective/rlhf_reward": 13.498762130737305, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008075715741142631, + "policy/clipfrac_avg": 0.011311197653412819, + "policy/entropy_avg": 0.1630105972290039, + "step": 931, + "val/clipfrac_avg": 0.003885046113282442, + "val/num_eos_tokens": 302, + "val/ratio": 0.9890309572219849, + "val/ratio_var": 5.2104482165304944e-05 + }, + { + "episode": 14912, + "epoch": 0.7531313131313131, + "eps": 0, + "loss/policy_avg": -0.30045461654663086, + "loss/value_avg": 0.872713565826416, + "lr": 1.5893570022344124e-06, + "objective/entropy": -78.21574401855469, + "objective/kl": 32.467742919921875, + "objective/non_score_reward": -1.623387098312378, + "objective/rlhf_reward": 14.372706413269043, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008647768991068006, + "policy/clipfrac_avg": 0.005656212102621794, + "policy/entropy_avg": 0.17721939086914062, + "step": 932, + "val/clipfrac_avg": 0.0015919646248221397, + "val/num_eos_tokens": 12, + "val/ratio": 0.9885766506195068, + "val/ratio_var": 6.424664752557874e-05 + }, + { + "episode": 14928, + "epoch": 0.7539393939393939, + "eps": 0, + "loss/policy_avg": -0.012097490951418877, + "loss/value_avg": 0.7004050016403198, + "lr": 1.5796020333532696e-06, + "objective/entropy": -61.095802307128906, + "objective/kl": 36.504150390625, + "objective/non_score_reward": -1.8252074718475342, + "objective/rlhf_reward": 12.393542289733887, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008130621863529086, + "policy/clipfrac_avg": 0.005108194425702095, + "policy/entropy_avg": 0.16521072387695312, + "step": 933, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 182, + "val/ratio": 0.9887269735336304, + "val/ratio_var": 6.136602314654738e-05 + }, + { + "episode": 14944, + "epoch": 0.7547474747474747, + "eps": 0, + "loss/policy_avg": 0.03149443119764328, + "loss/value_avg": 0.3308320641517639, + "lr": 1.5698714741182459e-06, + "objective/entropy": -110.07270812988281, + "objective/kl": 34.973388671875, + "objective/non_score_reward": -1.7486693859100342, + "objective/rlhf_reward": 14.548205375671387, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00091616774443537, + "policy/clipfrac_avg": 0.006772395223379135, + "policy/entropy_avg": 0.17972993850708008, + "step": 934, + "val/clipfrac_avg": 0.0015980113530531526, + "val/num_eos_tokens": 350, + "val/ratio": 0.9878275394439697, + "val/ratio_var": 8.512718341080472e-05 + }, + { + "episode": 14960, + "epoch": 0.7555555555555555, + "eps": 0, + "loss/policy_avg": -0.03353989124298096, + "loss/value_avg": 0.4103987216949463, + "lr": 1.5601653939714073e-06, + "objective/entropy": -25.034250259399414, + "objective/kl": 39.53004837036133, + "objective/non_score_reward": -1.976502537727356, + "objective/rlhf_reward": 14.281310081481934, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102006527595222, + "policy/clipfrac_avg": 0.00497299712151289, + "policy/entropy_avg": 0.19387245178222656, + "step": 935, + "val/clipfrac_avg": 0.0242628026753664, + "val/num_eos_tokens": 73, + "val/ratio": 0.9875220060348511, + "val/ratio_var": 6.656184268649668e-05 + }, + { + "episode": 14976, + "epoch": 0.7563636363636363, + "eps": 0, + "loss/policy_avg": -0.1474141776561737, + "loss/value_avg": 0.527273416519165, + "lr": 1.5504838621801272e-06, + "objective/entropy": -60.77607345581055, + "objective/kl": 37.98503494262695, + "objective/non_score_reward": -1.8992516994476318, + "objective/rlhf_reward": 13.171060562133789, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007948282873257995, + "policy/clipfrac_avg": 0.002496684668585658, + "policy/entropy_avg": 0.1510624885559082, + "step": 936, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 3.4012387914117426e-05 + }, + { + "episode": 14992, + "epoch": 0.7571717171717172, + "eps": 0, + "loss/policy_avg": 0.1368754655122757, + "loss/value_avg": 0.6238852143287659, + "lr": 1.5408269478365883e-06, + "objective/entropy": -60.082557678222656, + "objective/kl": 41.95027160644531, + "objective/non_score_reward": -2.0975136756896973, + "objective/rlhf_reward": 12.652486801147461, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009200765052810311, + "policy/clipfrac_avg": 0.0059701185673475266, + "policy/entropy_avg": 0.17743873596191406, + "step": 937, + "val/clipfrac_avg": 0.012415202334523201, + "val/num_eos_tokens": 54, + "val/ratio": 0.9879815578460693, + "val/ratio_var": 4.387104490888305e-05 + }, + { + "episode": 15008, + "epoch": 0.757979797979798, + "eps": 0, + "loss/policy_avg": -0.17327642440795898, + "loss/value_avg": 0.538246750831604, + "lr": 1.5311947198572918e-06, + "objective/entropy": -112.668701171875, + "objective/kl": 35.57097625732422, + "objective/non_score_reward": -1.7785489559173584, + "objective/rlhf_reward": 13.416763305664062, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008795931353233755, + "policy/clipfrac_avg": 0.005144854076206684, + "policy/entropy_avg": 0.16583538055419922, + "step": 938, + "val/clipfrac_avg": 0.047972679138183594, + "val/num_eos_tokens": 136, + "val/ratio": 0.9902178645133972, + "val/ratio_var": 5.2144809160381556e-05 + }, + { + "episode": 15024, + "epoch": 0.7587878787878788, + "eps": 0, + "loss/policy_avg": 0.5920745134353638, + "loss/value_avg": 1.1476694345474243, + "lr": 1.5215872469825682e-06, + "objective/entropy": -72.79998779296875, + "objective/kl": 35.88062286376953, + "objective/non_score_reward": -1.7940311431884766, + "objective/rlhf_reward": 14.46378231048584, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008520716801285744, + "policy/clipfrac_avg": 0.0030313697643578053, + "policy/entropy_avg": 0.19948351383209229, + "step": 939, + "val/clipfrac_avg": 0.021162573248147964, + "val/num_eos_tokens": 210, + "val/ratio": 0.9882235527038574, + "val/ratio_var": 6.201990618137643e-05 + }, + { + "episode": 15040, + "epoch": 0.7595959595959596, + "eps": 0, + "loss/policy_avg": -0.15639176964759827, + "loss/value_avg": 0.8813156485557556, + "lr": 1.5120045977760784e-06, + "objective/entropy": -91.86679077148438, + "objective/kl": 42.35734176635742, + "objective/non_score_reward": -2.1178669929504395, + "objective/rlhf_reward": 15.264945983886719, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0010741858277469873, + "policy/clipfrac_avg": 0.00790454912930727, + "policy/entropy_avg": 0.19293498992919922, + "step": 940, + "val/clipfrac_avg": 0.015625, + "val/num_eos_tokens": 9, + "val/ratio": 0.986987829208374, + "val/ratio_var": 6.426944310078397e-05 + }, + { + "episode": 15056, + "epoch": 0.7604040404040404, + "eps": 0, + "loss/policy_avg": 0.2165047824382782, + "loss/value_avg": 0.34554335474967957, + "lr": 1.502446840624331e-06, + "objective/entropy": -62.31670379638672, + "objective/kl": 31.252155303955078, + "objective/non_score_reward": -1.562608003616333, + "objective/rlhf_reward": 14.156142234802246, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007184218848124146, + "policy/clipfrac_avg": 0.0035393601283431053, + "policy/entropy_avg": 0.14359378814697266, + "step": 941, + "val/clipfrac_avg": 0.013209432363510132, + "val/num_eos_tokens": 36, + "val/ratio": 0.9896091222763062, + "val/ratio_var": 5.599396536126733e-05 + }, + { + "episode": 15072, + "epoch": 0.7612121212121212, + "eps": 0, + "loss/policy_avg": -0.030075984075665474, + "loss/value_avg": 0.45760178565979004, + "lr": 1.4929140437361916e-06, + "objective/entropy": -109.5699691772461, + "objective/kl": 38.075103759765625, + "objective/non_score_reward": -1.9037551879882812, + "objective/rlhf_reward": 14.713432312011719, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010412151459604502, + "policy/clipfrac_avg": 0.009930627420544624, + "policy/entropy_avg": 0.18667316436767578, + "step": 942, + "val/clipfrac_avg": 0.014139890670776367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9863598346710205, + "val/ratio_var": 0.00010634912905516103 + }, + { + "episode": 15088, + "epoch": 0.762020202020202, + "eps": 0, + "loss/policy_avg": 0.12722979485988617, + "loss/value_avg": 0.5696362853050232, + "lr": 1.4834062751424018e-06, + "objective/entropy": -121.374267578125, + "objective/kl": 34.23678207397461, + "objective/non_score_reward": -1.711839199066162, + "objective/rlhf_reward": 14.038161277770996, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007155380444601178, + "policy/clipfrac_avg": 0.00446352269500494, + "policy/entropy_avg": 0.13922500610351562, + "step": 943, + "val/clipfrac_avg": 0.001972501166164875, + "val/num_eos_tokens": 493, + "val/ratio": 0.9906710386276245, + "val/ratio_var": 6.088761438149959e-05 + }, + { + "episode": 15104, + "epoch": 0.7628282828282829, + "eps": 0, + "loss/policy_avg": 0.24459479749202728, + "loss/value_avg": 0.997427225112915, + "lr": 1.4739236026950838e-06, + "objective/entropy": -117.67542266845703, + "objective/kl": 42.60675811767578, + "objective/non_score_reward": -2.130337953567505, + "objective/rlhf_reward": 13.854036331176758, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007362811593338847, + "policy/clipfrac_avg": 0.009282913990318775, + "policy/entropy_avg": 0.15163874626159668, + "step": 944, + "val/clipfrac_avg": 0.019384628161787987, + "val/num_eos_tokens": 404, + "val/ratio": 0.9907106161117554, + "val/ratio_var": 5.1745548262260854e-05 + }, + { + "episode": 15120, + "epoch": 0.7636363636363637, + "eps": 0, + "loss/policy_avg": 0.019024059176445007, + "loss/value_avg": 0.2505372166633606, + "lr": 1.4644660940672628e-06, + "objective/entropy": -72.49942779541016, + "objective/kl": 36.86358642578125, + "objective/non_score_reward": -1.8431792259216309, + "objective/rlhf_reward": 14.227133750915527, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009655736503191292, + "policy/clipfrac_avg": 0.0035610971972346306, + "policy/entropy_avg": 0.19275665283203125, + "step": 945, + "val/clipfrac_avg": 0.0008617835119366646, + "val/num_eos_tokens": 33, + "val/ratio": 0.9873601198196411, + "val/ratio_var": 7.99532572273165e-05 + }, + { + "episode": 15136, + "epoch": 0.7644444444444445, + "eps": 0, + "loss/policy_avg": -0.24745053052902222, + "loss/value_avg": 0.4052670896053314, + "lr": 1.455033816752383e-06, + "objective/entropy": -106.38670349121094, + "objective/kl": 30.419466018676758, + "objective/non_score_reward": -1.5209734439849854, + "objective/rlhf_reward": 15.338401794433594, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009057885035872459, + "policy/clipfrac_avg": 0.009060706943273544, + "policy/entropy_avg": 0.161224365234375, + "step": 946, + "val/clipfrac_avg": 0.038856737315654755, + "val/num_eos_tokens": 409, + "val/ratio": 0.9891325235366821, + "val/ratio_var": 9.022479935083538e-05 + }, + { + "episode": 15152, + "epoch": 0.7652525252525253, + "eps": 0, + "loss/policy_avg": -0.051016077399253845, + "loss/value_avg": 0.5337303876876831, + "lr": 1.4456268380638262e-06, + "objective/entropy": -104.14939880371094, + "objective/kl": 36.21892547607422, + "objective/non_score_reward": -1.8109462261199951, + "objective/rlhf_reward": 15.282804489135742, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009193423902615905, + "policy/clipfrac_avg": 0.010075472295284271, + "policy/entropy_avg": 0.16665363311767578, + "step": 947, + "val/clipfrac_avg": 0.006654549390077591, + "val/num_eos_tokens": 424, + "val/ratio": 0.9888213276863098, + "val/ratio_var": 8.222387987188995e-05 + }, + { + "episode": 15168, + "epoch": 0.7660606060606061, + "eps": 0, + "loss/policy_avg": 0.5708428025245667, + "loss/value_avg": 0.9156904220581055, + "lr": 1.4362452251344283e-06, + "objective/entropy": -107.8341064453125, + "objective/kl": 37.6370849609375, + "objective/non_score_reward": -1.8818542957305908, + "objective/rlhf_reward": 11.461895942687988, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008800949435681105, + "policy/clipfrac_avg": 0.018943525850772858, + "policy/entropy_avg": 0.18418407440185547, + "step": 948, + "val/clipfrac_avg": 0.0006793478387407959, + "val/num_eos_tokens": 297, + "val/ratio": 0.9880671501159668, + "val/ratio_var": 7.767622446408495e-05 + }, + { + "episode": 15184, + "epoch": 0.7668686868686869, + "eps": 0, + "loss/policy_avg": -0.028068287298083305, + "loss/value_avg": 0.6219231486320496, + "lr": 1.4268890449160034e-06, + "objective/entropy": -11.566868782043457, + "objective/kl": 34.966495513916016, + "objective/non_score_reward": -1.7483247518539429, + "objective/rlhf_reward": 14.603238105773926, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0011735728476196527, + "policy/clipfrac_avg": 0.004941962193697691, + "policy/entropy_avg": 0.20639419555664062, + "step": 949, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 32, + "val/ratio": 0.9847333431243896, + "val/ratio_var": 3.736758299055509e-05 + }, + { + "episode": 15200, + "epoch": 0.7676767676767676, + "eps": 0, + "loss/policy_avg": -0.15076535940170288, + "loss/value_avg": 0.2775721549987793, + "lr": 1.4175583641788698e-06, + "objective/entropy": -80.4765625, + "objective/kl": 31.678754806518555, + "objective/non_score_reward": -1.583937644958496, + "objective/rlhf_reward": 13.853562355041504, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0006830325582996011, + "policy/clipfrac_avg": 0.00417131744325161, + "policy/entropy_avg": 0.1293621063232422, + "step": 950, + "val/clipfrac_avg": 0.0393822118639946, + "val/num_eos_tokens": 257, + "val/ratio": 0.9911376237869263, + "val/ratio_var": 2.9909415388829075e-05 + }, + { + "episode": 15216, + "epoch": 0.7684848484848484, + "eps": 0, + "loss/policy_avg": -0.04640840366482735, + "loss/value_avg": 0.3922756314277649, + "lr": 1.4082532495113627e-06, + "objective/entropy": -84.18096160888672, + "objective/kl": 38.69972229003906, + "objective/non_score_reward": -1.9349862337112427, + "objective/rlhf_reward": 15.416576385498047, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.000798642635345459, + "policy/clipfrac_avg": 0.0038171219639480114, + "policy/entropy_avg": 0.1776995062828064, + "step": 951, + "val/clipfrac_avg": 0.0032731681130826473, + "val/num_eos_tokens": 326, + "val/ratio": 0.9889781475067139, + "val/ratio_var": 5.715083170798607e-05 + }, + { + "episode": 15232, + "epoch": 0.7692929292929293, + "eps": 0, + "loss/policy_avg": -0.12606917321681976, + "loss/value_avg": 0.3175104856491089, + "lr": 1.3989737673193682e-06, + "objective/entropy": -60.43904113769531, + "objective/kl": 39.91594314575195, + "objective/non_score_reward": -1.9957971572875977, + "objective/rlhf_reward": 14.262016296386719, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009779632091522217, + "policy/clipfrac_avg": 0.004585625603795052, + "policy/entropy_avg": 0.2008209228515625, + "step": 952, + "val/clipfrac_avg": 0.030662208795547485, + "val/num_eos_tokens": 422, + "val/ratio": 0.9862244129180908, + "val/ratio_var": 8.592133963247761e-05 + }, + { + "episode": 15248, + "epoch": 0.7701010101010101, + "eps": 0, + "loss/policy_avg": 0.08768834173679352, + "loss/value_avg": 0.327705055475235, + "lr": 1.389719983825844e-06, + "objective/entropy": -67.14826965332031, + "objective/kl": 46.73610305786133, + "objective/non_score_reward": -2.3368051052093506, + "objective/rlhf_reward": 14.29600715637207, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011253968114033341, + "policy/clipfrac_avg": 0.006955197546631098, + "policy/entropy_avg": 0.20795369148254395, + "step": 953, + "val/clipfrac_avg": 8.926542068365961e-05, + "val/num_eos_tokens": 421, + "val/ratio": 0.9854748249053955, + "val/ratio_var": 6.634320016019046e-05 + }, + { + "episode": 15264, + "epoch": 0.7709090909090909, + "eps": 0, + "loss/policy_avg": 0.17172515392303467, + "loss/value_avg": 0.5182530283927917, + "lr": 1.3804919650703551e-06, + "objective/entropy": -52.92948913574219, + "objective/kl": 39.47554016113281, + "objective/non_score_reward": -1.9737770557403564, + "objective/rlhf_reward": 15.323097229003906, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009001094149425626, + "policy/clipfrac_avg": 0.005291195120662451, + "policy/entropy_avg": 0.18181610107421875, + "step": 954, + "val/clipfrac_avg": 0.006615057587623596, + "val/num_eos_tokens": 490, + "val/ratio": 0.9876183271408081, + "val/ratio_var": 5.796940240543336e-05 + }, + { + "episode": 15280, + "epoch": 0.7717171717171717, + "eps": 0, + "loss/policy_avg": 0.3471048176288605, + "loss/value_avg": 0.7186154723167419, + "lr": 1.3712897769085903e-06, + "objective/entropy": -93.25178527832031, + "objective/kl": 41.423980712890625, + "objective/non_score_reward": -2.0711989402770996, + "objective/rlhf_reward": 13.155363082885742, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009469924843870103, + "policy/clipfrac_avg": 0.004164956044405699, + "policy/entropy_avg": 0.18822860717773438, + "step": 955, + "val/clipfrac_avg": 0.04515676945447922, + "val/num_eos_tokens": 611, + "val/ratio": 0.9877511262893677, + "val/ratio_var": 7.300172001123428e-05 + }, + { + "episode": 15296, + "epoch": 0.7725252525252525, + "eps": 0, + "loss/policy_avg": 0.24007092416286469, + "loss/value_avg": 0.4516885280609131, + "lr": 1.3621134850119012e-06, + "objective/entropy": -112.50715637207031, + "objective/kl": 31.20224380493164, + "objective/non_score_reward": -1.5601121187210083, + "objective/rlhf_reward": 13.799263000488281, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007952903397381306, + "policy/clipfrac_avg": 0.008523119613528252, + "policy/entropy_avg": 0.1457352638244629, + "step": 956, + "val/clipfrac_avg": 0.06015515699982643, + "val/num_eos_tokens": 383, + "val/ratio": 0.9895913600921631, + "val/ratio_var": 7.848602399462834e-05 + }, + { + "episode": 15312, + "epoch": 0.7733333333333333, + "eps": 0, + "loss/policy_avg": -0.03721753507852554, + "loss/value_avg": 0.241817444562912, + "lr": 1.3529631548668298e-06, + "objective/entropy": -82.6330337524414, + "objective/kl": 44.714515686035156, + "objective/non_score_reward": -2.2357258796691895, + "objective/rlhf_reward": 13.818962097167969, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0006799673428758979, + "policy/clipfrac_avg": 0.003063515294343233, + "policy/entropy_avg": 0.1481151580810547, + "step": 957, + "val/clipfrac_avg": 9.245562250725925e-05, + "val/num_eos_tokens": 341, + "val/ratio": 0.9905509948730469, + "val/ratio_var": 4.010691918665543e-05 + }, + { + "episode": 15328, + "epoch": 0.7741414141414141, + "eps": 0, + "loss/policy_avg": -0.1448572874069214, + "loss/value_avg": 0.6361722946166992, + "lr": 1.3438388517746431e-06, + "objective/entropy": -75.73651885986328, + "objective/kl": 32.26376724243164, + "objective/non_score_reward": -1.6131882667541504, + "objective/rlhf_reward": 13.550874710083008, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010009508114308119, + "policy/clipfrac_avg": 0.004912985023111105, + "policy/entropy_avg": 0.190643310546875, + "step": 958, + "val/clipfrac_avg": 0.008336715400218964, + "val/num_eos_tokens": 329, + "val/ratio": 0.9864659905433655, + "val/ratio_var": 9.685769327916205e-05 + }, + { + "episode": 15344, + "epoch": 0.7749494949494949, + "eps": 0, + "loss/policy_avg": 0.11857907474040985, + "loss/value_avg": 0.8868049383163452, + "lr": 1.3347406408508695e-06, + "objective/entropy": -90.25643920898438, + "objective/kl": 39.02738952636719, + "objective/non_score_reward": -1.9513695240020752, + "objective/rlhf_reward": 14.954880714416504, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009018229902721941, + "policy/clipfrac_avg": 0.004778256639838219, + "policy/entropy_avg": 0.1796565055847168, + "step": 959, + "val/clipfrac_avg": 0.02706507220864296, + "val/num_eos_tokens": 289, + "val/ratio": 0.9882634878158569, + "val/ratio_var": 9.323142148787156e-05 + }, + { + "episode": 15360, + "epoch": 0.7757575757575758, + "eps": 0, + "loss/policy_avg": -0.22503328323364258, + "loss/value_avg": 0.4810031056404114, + "lr": 1.3256685870248227e-06, + "objective/entropy": -101.29171752929688, + "objective/kl": 37.34764099121094, + "objective/non_score_reward": -1.8673819303512573, + "objective/rlhf_reward": 14.32011890411377, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008543966105207801, + "policy/clipfrac_avg": 0.0047050644643604755, + "policy/entropy_avg": 0.17150402069091797, + "step": 960, + "val/clipfrac_avg": 0.0005405173869803548, + "val/num_eos_tokens": 130, + "val/ratio": 0.9882644414901733, + "val/ratio_var": 6.539086461998522e-05 + }, + { + "episode": 15376, + "epoch": 0.7765656565656566, + "eps": 0, + "loss/policy_avg": -0.0476013720035553, + "loss/value_avg": 0.682388961315155, + "lr": 1.3166227550391542e-06, + "objective/entropy": -97.00421142578125, + "objective/kl": 42.212120056152344, + "objective/non_score_reward": -2.1106061935424805, + "objective/rlhf_reward": 13.99876880645752, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000946523156017065, + "policy/clipfrac_avg": 0.0059087686240673065, + "policy/entropy_avg": 0.19147014617919922, + "step": 961, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 224, + "val/ratio": 0.9874411225318909, + "val/ratio_var": 5.3472565923584625e-05 + }, + { + "episode": 15392, + "epoch": 0.7773737373737374, + "eps": 0, + "loss/policy_avg": -0.26426494121551514, + "loss/value_avg": 0.47233304381370544, + "lr": 1.307603209449378e-06, + "objective/entropy": -60.28771209716797, + "objective/kl": 39.21135330200195, + "objective/non_score_reward": -1.9605677127838135, + "objective/rlhf_reward": 15.351932525634766, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009600625489838421, + "policy/clipfrac_avg": 0.0035959878005087376, + "policy/entropy_avg": 0.18239498138427734, + "step": 962, + "val/clipfrac_avg": 0.012040323577821255, + "val/num_eos_tokens": 296, + "val/ratio": 0.9875339269638062, + "val/ratio_var": 6.485576159320772e-05 + }, + { + "episode": 15408, + "epoch": 0.7781818181818182, + "eps": 0, + "loss/policy_avg": 0.04519583284854889, + "loss/value_avg": 0.7595553398132324, + "lr": 1.298610014623423e-06, + "objective/entropy": -146.63661193847656, + "objective/kl": 38.37220001220703, + "objective/non_score_reward": -1.9186100959777832, + "objective/rlhf_reward": 13.354827880859375, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007201317930594087, + "policy/clipfrac_avg": 0.0022899312898516655, + "policy/entropy_avg": 0.1562885046005249, + "step": 963, + "val/clipfrac_avg": 8.292270649690181e-05, + "val/num_eos_tokens": 521, + "val/ratio": 0.9906989336013794, + "val/ratio_var": 6.185618985909969e-05 + }, + { + "episode": 15424, + "epoch": 0.778989898989899, + "eps": 0, + "loss/policy_avg": -0.14623457193374634, + "loss/value_avg": 0.35101252794265747, + "lr": 1.2896432347411553e-06, + "objective/entropy": -66.08819580078125, + "objective/kl": 36.48185348510742, + "objective/non_score_reward": -1.8240927457809448, + "objective/rlhf_reward": 14.261844635009766, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009832159848883748, + "policy/clipfrac_avg": 0.006735666189342737, + "policy/entropy_avg": 0.19070911407470703, + "step": 964, + "val/clipfrac_avg": 0.0030241934582591057, + "val/num_eos_tokens": 421, + "val/ratio": 0.9872285723686218, + "val/ratio_var": 7.909470878075808e-05 + }, + { + "episode": 15440, + "epoch": 0.7797979797979798, + "eps": 0, + "loss/policy_avg": -0.1669466346502304, + "loss/value_avg": 0.6373587846755981, + "lr": 1.2807029337939403e-06, + "objective/entropy": -48.76942825317383, + "objective/kl": 36.51701354980469, + "objective/non_score_reward": -1.8258507251739502, + "objective/rlhf_reward": 13.861648559570312, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.000836203049402684, + "policy/clipfrac_avg": 0.006686762906610966, + "policy/entropy_avg": 0.17627906799316406, + "step": 965, + "val/clipfrac_avg": 0.009943181648850441, + "val/num_eos_tokens": 11, + "val/ratio": 0.9892607927322388, + "val/ratio_var": 5.405569390859455e-05 + }, + { + "episode": 15456, + "epoch": 0.7806060606060606, + "eps": 0, + "loss/policy_avg": -0.005087921395897865, + "loss/value_avg": 0.2712212800979614, + "lr": 1.2717891755841722e-06, + "objective/entropy": -59.34945297241211, + "objective/kl": 42.5623893737793, + "objective/non_score_reward": -2.128119468688965, + "objective/rlhf_reward": 13.918755531311035, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008710323600098491, + "policy/clipfrac_avg": 0.010207775980234146, + "policy/entropy_avg": 0.1739978790283203, + "step": 966, + "val/clipfrac_avg": 0.010802468284964561, + "val/num_eos_tokens": 246, + "val/ratio": 0.9884668588638306, + "val/ratio_var": 4.567186624626629e-05 + }, + { + "episode": 15472, + "epoch": 0.7814141414141414, + "eps": 0, + "loss/policy_avg": -0.5036620497703552, + "loss/value_avg": 0.39513152837753296, + "lr": 1.2629020237248241e-06, + "objective/entropy": -136.59652709960938, + "objective/kl": 29.273542404174805, + "objective/non_score_reward": -1.463677167892456, + "objective/rlhf_reward": 14.083197593688965, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0007826394285075366, + "policy/clipfrac_avg": 0.006299240980297327, + "policy/entropy_avg": 0.15417519211769104, + "step": 967, + "val/clipfrac_avg": 0.03271849453449249, + "val/num_eos_tokens": 249, + "val/ratio": 0.9902065992355347, + "val/ratio_var": 8.734448783798143e-05 + }, + { + "episode": 15488, + "epoch": 0.7822222222222223, + "eps": 0, + "loss/policy_avg": -0.04834941402077675, + "loss/value_avg": 0.49886178970336914, + "lr": 1.2540415416389917e-06, + "objective/entropy": -29.31902503967285, + "objective/kl": 38.655548095703125, + "objective/non_score_reward": -1.9327776432037354, + "objective/rlhf_reward": 13.864096641540527, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008980755810625851, + "policy/clipfrac_avg": 0.003656789194792509, + "policy/entropy_avg": 0.1847667694091797, + "step": 968, + "val/clipfrac_avg": 5.3879310144111514e-05, + "val/num_eos_tokens": 10, + "val/ratio": 0.9879658818244934, + "val/ratio_var": 2.3956074073794298e-05 + }, + { + "episode": 15504, + "epoch": 0.7830303030303031, + "eps": 0, + "loss/policy_avg": -0.2217503786087036, + "loss/value_avg": 0.4100498557090759, + "lr": 1.2452077925594435e-06, + "objective/entropy": -206.95504760742188, + "objective/kl": 41.82144546508789, + "objective/non_score_reward": -2.0910723209381104, + "objective/rlhf_reward": 13.076896667480469, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008509804611094296, + "policy/clipfrac_avg": 0.0047798133455216885, + "policy/entropy_avg": 0.15673446655273438, + "step": 969, + "val/clipfrac_avg": 0.00021062050655018538, + "val/num_eos_tokens": 481, + "val/ratio": 0.9887630939483643, + "val/ratio_var": 6.604092777706683e-05 + }, + { + "episode": 15520, + "epoch": 0.7838383838383839, + "eps": 0, + "loss/policy_avg": -0.02120068296790123, + "loss/value_avg": 0.6901993751525879, + "lr": 1.23640083952817e-06, + "objective/entropy": -40.900238037109375, + "objective/kl": 38.07307052612305, + "objective/non_score_reward": -1.9036535024642944, + "objective/rlhf_reward": 14.377596855163574, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010727059561759233, + "policy/clipfrac_avg": 0.004337977152317762, + "policy/entropy_avg": 0.1984100341796875, + "step": 970, + "val/clipfrac_avg": 0.02858383022248745, + "val/num_eos_tokens": 535, + "val/ratio": 0.9863028526306152, + "val/ratio_var": 8.229557715822011e-05 + }, + { + "episode": 15536, + "epoch": 0.7846464646464646, + "eps": 0, + "loss/policy_avg": -0.09917282313108444, + "loss/value_avg": 0.3991038203239441, + "lr": 1.2276207453959283e-06, + "objective/entropy": -77.6534652709961, + "objective/kl": 39.35569381713867, + "objective/non_score_reward": -1.9677847623825073, + "objective/rlhf_reward": 13.907215118408203, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008703683270141482, + "policy/clipfrac_avg": 0.0043828776106238365, + "policy/entropy_avg": 0.17210960388183594, + "step": 971, + "val/clipfrac_avg": 0.00012400794366840273, + "val/num_eos_tokens": 175, + "val/ratio": 0.988074541091919, + "val/ratio_var": 5.318454350344837e-05 + }, + { + "episode": 15552, + "epoch": 0.7854545454545454, + "eps": 0, + "loss/policy_avg": 0.0814996063709259, + "loss/value_avg": 0.6175538897514343, + "lr": 1.2188675728217986e-06, + "objective/entropy": -16.92827606201172, + "objective/kl": 41.14687728881836, + "objective/non_score_reward": -2.0573439598083496, + "objective/rlhf_reward": 15.442655563354492, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010900306515395641, + "policy/clipfrac_avg": 0.00866631604731083, + "policy/entropy_avg": 0.2063446044921875, + "step": 972, + "val/clipfrac_avg": 0.009025804698467255, + "val/num_eos_tokens": 174, + "val/ratio": 0.9854716062545776, + "val/ratio_var": 2.7480951757752337e-05 + }, + { + "episode": 15568, + "epoch": 0.7862626262626262, + "eps": 0, + "loss/policy_avg": 0.3593114912509918, + "loss/value_avg": 0.7372395396232605, + "lr": 1.2101413842727345e-06, + "objective/entropy": -63.132266998291016, + "objective/kl": 45.04676818847656, + "objective/non_score_reward": -2.252338409423828, + "objective/rlhf_reward": 14.450786590576172, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009616734459996223, + "policy/clipfrac_avg": 0.0046102688647806644, + "policy/entropy_avg": 0.1856832504272461, + "step": 973, + "val/clipfrac_avg": 0.016889449208974838, + "val/num_eos_tokens": 419, + "val/ratio": 0.9877485036849976, + "val/ratio_var": 6.443963502533734e-05 + }, + { + "episode": 15584, + "epoch": 0.787070707070707, + "eps": 0, + "loss/policy_avg": 0.14888805150985718, + "loss/value_avg": 0.7544107437133789, + "lr": 1.201442242023122e-06, + "objective/entropy": -98.38099670410156, + "objective/kl": 41.9028434753418, + "objective/non_score_reward": -2.095142126083374, + "objective/rlhf_reward": 12.186107635498047, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009328118758276105, + "policy/clipfrac_avg": 0.004906289279460907, + "policy/entropy_avg": 0.17488479614257812, + "step": 974, + "val/clipfrac_avg": 0.08914318680763245, + "val/num_eos_tokens": 193, + "val/ratio": 0.9888035655021667, + "val/ratio_var": 6.763465353287756e-05 + }, + { + "episode": 15600, + "epoch": 0.7878787878787878, + "eps": 0, + "loss/policy_avg": -0.05416979640722275, + "loss/value_avg": 0.5626612901687622, + "lr": 1.1927702081543279e-06, + "objective/entropy": -65.54731750488281, + "objective/kl": 33.80537796020508, + "objective/non_score_reward": -1.6902689933776855, + "objective/rlhf_reward": 13.434730529785156, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010051055578514934, + "policy/clipfrac_avg": 0.005396673921495676, + "policy/entropy_avg": 0.19644975662231445, + "step": 975, + "val/clipfrac_avg": 0.02293882891535759, + "val/num_eos_tokens": 539, + "val/ratio": 0.9861206412315369, + "val/ratio_var": 7.48153543099761e-05 + }, + { + "episode": 15616, + "epoch": 0.7886868686868687, + "eps": 0, + "loss/policy_avg": 0.05607307702302933, + "loss/value_avg": 0.48446062207221985, + "lr": 1.1841253445542605e-06, + "objective/entropy": -29.001220703125, + "objective/kl": 36.37082290649414, + "objective/non_score_reward": -1.8185412883758545, + "objective/rlhf_reward": 12.142396926879883, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0011397147318348289, + "policy/clipfrac_avg": 0.0031259057577699423, + "policy/entropy_avg": 0.22170257568359375, + "step": 976, + "val/clipfrac_avg": 0.005424283444881439, + "val/num_eos_tokens": 387, + "val/ratio": 0.9855947494506836, + "val/ratio_var": 6.33225790807046e-05 + }, + { + "episode": 15632, + "epoch": 0.7894949494949495, + "eps": 0, + "loss/policy_avg": -0.15400168299674988, + "loss/value_avg": 0.8165977001190186, + "lr": 1.1755077129169284e-06, + "objective/entropy": -60.56960678100586, + "objective/kl": 37.47486114501953, + "objective/non_score_reward": -1.8737430572509766, + "objective/rlhf_reward": 11.046178817749023, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0009815995581448078, + "policy/clipfrac_avg": 0.005213981028646231, + "policy/entropy_avg": 0.21601486206054688, + "step": 977, + "val/clipfrac_avg": 0.022167012095451355, + "val/num_eos_tokens": 93, + "val/ratio": 0.9861747026443481, + "val/ratio_var": 7.86496966611594e-05 + }, + { + "episode": 15648, + "epoch": 0.7903030303030303, + "eps": 0, + "loss/policy_avg": -0.1725654900074005, + "loss/value_avg": 0.487238347530365, + "lr": 1.166917374742e-06, + "objective/entropy": -56.807586669921875, + "objective/kl": 37.225948333740234, + "objective/non_score_reward": -1.861297607421875, + "objective/rlhf_reward": 14.388702392578125, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102808338124305, + "policy/clipfrac_avg": 0.0037505230866372585, + "policy/entropy_avg": 0.20328998565673828, + "step": 978, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 128, + "val/ratio": 0.9871898293495178, + "val/ratio_var": 9.372083150083199e-05 + }, + { + "episode": 15664, + "epoch": 0.7911111111111111, + "eps": 0, + "loss/policy_avg": -0.2764543294906616, + "loss/value_avg": 0.33185577392578125, + "lr": 1.158354391334362e-06, + "objective/entropy": -78.30638122558594, + "objective/kl": 34.59331512451172, + "objective/non_score_reward": -1.729665756225586, + "objective/rlhf_reward": 15.489084243774414, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008904373971745372, + "policy/clipfrac_avg": 0.00305290799587965, + "policy/entropy_avg": 0.17419052124023438, + "step": 979, + "val/clipfrac_avg": 0.09470640122890472, + "val/num_eos_tokens": 367, + "val/ratio": 0.9881394505500793, + "val/ratio_var": 6.588637916138396e-05 + }, + { + "episode": 15680, + "epoch": 0.7919191919191919, + "eps": 0, + "loss/policy_avg": 0.049546245485544205, + "loss/value_avg": 0.5992924571037292, + "lr": 1.1498188238036862e-06, + "objective/entropy": -146.6639862060547, + "objective/kl": 40.94211959838867, + "objective/non_score_reward": -2.0471062660217285, + "objective/rlhf_reward": 12.40992546081543, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008619745494797826, + "policy/clipfrac_avg": 0.007621705532073975, + "policy/entropy_avg": 0.15411758422851562, + "step": 980, + "val/clipfrac_avg": 0.09697519987821579, + "val/num_eos_tokens": 405, + "val/ratio": 0.9889684915542603, + "val/ratio_var": 5.157575287739746e-05 + }, + { + "episode": 15696, + "epoch": 0.7927272727272727, + "eps": 0, + "loss/policy_avg": 0.7101428508758545, + "loss/value_avg": 1.0345678329467773, + "lr": 1.141310733063991e-06, + "objective/entropy": -111.55574798583984, + "objective/kl": 35.79730224609375, + "objective/non_score_reward": -1.7898651361465454, + "objective/rlhf_reward": 13.764822006225586, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009224195964634418, + "policy/clipfrac_avg": 0.00732937827706337, + "policy/entropy_avg": 0.1737055778503418, + "step": 981, + "val/clipfrac_avg": 0.0015625000232830644, + "val/num_eos_tokens": 230, + "val/ratio": 0.9885663390159607, + "val/ratio_var": 7.395897409878671e-05 + }, + { + "episode": 15712, + "epoch": 0.7935353535353535, + "eps": 0, + "loss/policy_avg": -0.1941525638103485, + "loss/value_avg": 0.4167664647102356, + "lr": 1.132830179833207e-06, + "objective/entropy": -136.7578125, + "objective/kl": 41.060279846191406, + "objective/non_score_reward": -2.053014039993286, + "objective/rlhf_reward": 14.564172744750977, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008562670554965734, + "policy/clipfrac_avg": 0.005737176164984703, + "policy/entropy_avg": 0.1706523895263672, + "step": 982, + "val/clipfrac_avg": 0.006640523672103882, + "val/num_eos_tokens": 935, + "val/ratio": 0.9893268346786499, + "val/ratio_var": 6.982601917115971e-05 + }, + { + "episode": 15728, + "epoch": 0.7943434343434344, + "eps": 0, + "loss/policy_avg": -0.1428111046552658, + "loss/value_avg": 0.2174670249223709, + "lr": 1.1243772246327416e-06, + "objective/entropy": -109.03129577636719, + "objective/kl": 38.72102355957031, + "objective/non_score_reward": -1.9360511302947998, + "objective/rlhf_reward": 14.470198631286621, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000771217979490757, + "policy/clipfrac_avg": 0.002998888725414872, + "policy/entropy_avg": 0.16396713256835938, + "step": 983, + "val/clipfrac_avg": 0.020510124042630196, + "val/num_eos_tokens": 25, + "val/ratio": 0.989021897315979, + "val/ratio_var": 4.0093269490171224e-05 + }, + { + "episode": 15744, + "epoch": 0.7951515151515152, + "eps": 0, + "loss/policy_avg": 0.010445185005664825, + "loss/value_avg": 0.37888824939727783, + "lr": 1.1159519277870507e-06, + "objective/entropy": -6.635685920715332, + "objective/kl": 38.89892578125, + "objective/non_score_reward": -1.9449464082717896, + "objective/rlhf_reward": 14.7113037109375, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009513833792880177, + "policy/clipfrac_avg": 0.003157425671815872, + "policy/entropy_avg": 0.1973400115966797, + "step": 984, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 21, + "val/ratio": 0.9875931739807129, + "val/ratio_var": 5.873597910976969e-05 + }, + { + "episode": 15760, + "epoch": 0.795959595959596, + "eps": 0, + "loss/policy_avg": -0.28500208258628845, + "loss/value_avg": 0.4213254451751709, + "lr": 1.1075543494232049e-06, + "objective/entropy": -149.3587646484375, + "objective/kl": 31.575908660888672, + "objective/non_score_reward": -1.5787954330444336, + "objective/rlhf_reward": 15.75714111328125, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007162499241530895, + "policy/clipfrac_avg": 0.0078894617035985, + "policy/entropy_avg": 0.14484691619873047, + "step": 985, + "val/clipfrac_avg": 0.023666253313422203, + "val/num_eos_tokens": 155, + "val/ratio": 0.9908950328826904, + "val/ratio_var": 5.517086901818402e-05 + }, + { + "episode": 15776, + "epoch": 0.7967676767676768, + "eps": 0, + "loss/policy_avg": -0.016947008669376373, + "loss/value_avg": 0.3959222733974457, + "lr": 1.0991845494704673e-06, + "objective/entropy": -90.30593872070312, + "objective/kl": 44.143898010253906, + "objective/non_score_reward": -2.2071948051452637, + "objective/rlhf_reward": 12.292805671691895, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000890448980499059, + "policy/clipfrac_avg": 0.005722125992178917, + "policy/entropy_avg": 0.1587371826171875, + "step": 986, + "val/clipfrac_avg": 0.020597470924258232, + "val/num_eos_tokens": 327, + "val/ratio": 0.9887195825576782, + "val/ratio_var": 5.094557855045423e-05 + }, + { + "episode": 15792, + "epoch": 0.7975757575757576, + "eps": 0, + "loss/policy_avg": 0.022888287901878357, + "loss/value_avg": 0.36602234840393066, + "lr": 1.0908425876598512e-06, + "objective/entropy": -140.4935760498047, + "objective/kl": 42.647743225097656, + "objective/non_score_reward": -2.132387161254883, + "objective/rlhf_reward": 13.516050338745117, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008821362862363458, + "policy/clipfrac_avg": 0.005631787236779928, + "policy/entropy_avg": 0.1700587272644043, + "step": 987, + "val/clipfrac_avg": 0.012090040370821953, + "val/num_eos_tokens": 546, + "val/ratio": 0.9886399507522583, + "val/ratio_var": 5.682192204403691e-05 + }, + { + "episode": 15808, + "epoch": 0.7983838383838384, + "eps": 0, + "loss/policy_avg": 0.19011469185352325, + "loss/value_avg": 0.5114719271659851, + "lr": 1.082528523523713e-06, + "objective/entropy": -53.488460540771484, + "objective/kl": 36.984073638916016, + "objective/non_score_reward": -1.8492038249969482, + "objective/rlhf_reward": 14.049233436584473, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009789207251742482, + "policy/clipfrac_avg": 0.005884348414838314, + "policy/entropy_avg": 0.20690059661865234, + "step": 988, + "val/clipfrac_avg": 0.013353430666029453, + "val/num_eos_tokens": 353, + "val/ratio": 0.9872037768363953, + "val/ratio_var": 6.569022662006319e-05 + }, + { + "episode": 15824, + "epoch": 0.7991919191919192, + "eps": 0, + "loss/policy_avg": 0.1753331869840622, + "loss/value_avg": 0.3773725628852844, + "lr": 1.0742424163953096e-06, + "objective/entropy": -56.68682861328125, + "objective/kl": 37.34782409667969, + "objective/non_score_reward": -1.8673913478851318, + "objective/rlhf_reward": 13.050577163696289, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.000838876876514405, + "policy/clipfrac_avg": 0.0064134905114769936, + "policy/entropy_avg": 0.16259098052978516, + "step": 989, + "val/clipfrac_avg": 0.055415935814380646, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885395169258118, + "val/ratio_var": 8.253273699665442e-05 + }, + { + "episode": 15840, + "epoch": 0.8, + "eps": 0, + "loss/policy_avg": -0.013544011861085892, + "loss/value_avg": 0.5490726828575134, + "lr": 1.0659843254083919e-06, + "objective/entropy": -84.58714294433594, + "objective/kl": 34.28784942626953, + "objective/non_score_reward": -1.7143926620483398, + "objective/rlhf_reward": 13.20748233795166, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009669119026511908, + "policy/clipfrac_avg": 0.00355747202411294, + "policy/entropy_avg": 0.18613052368164062, + "step": 990, + "val/clipfrac_avg": 0.015838205814361572, + "val/num_eos_tokens": 115, + "val/ratio": 0.9877301454544067, + "val/ratio_var": 5.967232209513895e-05 + }, + { + "episode": 15856, + "epoch": 0.8008080808080809, + "eps": 0, + "loss/policy_avg": 0.17707303166389465, + "loss/value_avg": 0.24937717616558075, + "lr": 1.0577543094967613e-06, + "objective/entropy": -134.7545928955078, + "objective/kl": 35.57737731933594, + "objective/non_score_reward": -1.7788686752319336, + "objective/rlhf_reward": 15.010193824768066, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008520459523424506, + "policy/clipfrac_avg": 0.007036273833364248, + "policy/entropy_avg": 0.14983177185058594, + "step": 991, + "val/clipfrac_avg": 0.0005040322430431843, + "val/num_eos_tokens": 450, + "val/ratio": 0.9898406863212585, + "val/ratio_var": 4.8867601435631514e-05 + }, + { + "episode": 15872, + "epoch": 0.8016161616161617, + "eps": 0, + "loss/policy_avg": -0.1231779083609581, + "loss/value_avg": 0.27385565638542175, + "lr": 1.0495524273938735e-06, + "objective/entropy": -99.90908813476562, + "objective/kl": 37.30508804321289, + "objective/non_score_reward": -1.865254521369934, + "objective/rlhf_reward": 12.759745597839355, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008709601243026555, + "policy/clipfrac_avg": 0.004131123423576355, + "policy/entropy_avg": 0.17008209228515625, + "step": 992, + "val/clipfrac_avg": 0.01071952749043703, + "val/num_eos_tokens": 376, + "val/ratio": 0.9894505739212036, + "val/ratio_var": 6.905445479787886e-05 + }, + { + "episode": 15888, + "epoch": 0.8024242424242424, + "eps": 0, + "loss/policy_avg": -0.11878392100334167, + "loss/value_avg": 0.6012347340583801, + "lr": 1.041378737632402e-06, + "objective/entropy": -89.06427764892578, + "objective/kl": 39.19951629638672, + "objective/non_score_reward": -1.9599759578704834, + "objective/rlhf_reward": 14.532211303710938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009216031758114696, + "policy/clipfrac_avg": 0.00672511849552393, + "policy/entropy_avg": 0.1765270233154297, + "step": 993, + "val/clipfrac_avg": 0.0031250000465661287, + "val/num_eos_tokens": 74, + "val/ratio": 0.9882692098617554, + "val/ratio_var": 3.506496432237327e-05 + }, + { + "episode": 15904, + "epoch": 0.8032323232323232, + "eps": 0, + "loss/policy_avg": 0.11932457238435745, + "loss/value_avg": 0.28720468282699585, + "lr": 1.0332332985438248e-06, + "objective/entropy": -48.50525665283203, + "objective/kl": 37.94336700439453, + "objective/non_score_reward": -1.8971682786941528, + "objective/rlhf_reward": 15.641894340515137, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010036650346592069, + "policy/clipfrac_avg": 0.0061247204430401325, + "policy/entropy_avg": 0.18225669860839844, + "step": 994, + "val/clipfrac_avg": 0.01019021775573492, + "val/num_eos_tokens": 8, + "val/ratio": 0.9874961972236633, + "val/ratio_var": 5.3736544941784814e-05 + }, + { + "episode": 15920, + "epoch": 0.804040404040404, + "eps": 0, + "loss/policy_avg": 0.14462614059448242, + "loss/value_avg": 0.3247252106666565, + "lr": 1.0251161682580125e-06, + "objective/entropy": -108.57736206054688, + "objective/kl": 41.1889762878418, + "objective/non_score_reward": -2.059448719024658, + "objective/rlhf_reward": 14.182738304138184, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001018329057842493, + "policy/clipfrac_avg": 0.0034168935380876064, + "policy/entropy_avg": 0.20872879028320312, + "step": 995, + "val/clipfrac_avg": 0.00548538425937295, + "val/num_eos_tokens": 8, + "val/ratio": 0.9865425825119019, + "val/ratio_var": 8.348035044036806e-05 + }, + { + "episode": 15936, + "epoch": 0.8048484848484848, + "eps": 0, + "loss/policy_avg": 0.2817779779434204, + "loss/value_avg": 0.49794071912765503, + "lr": 1.0170274047028068e-06, + "objective/entropy": -43.08762741088867, + "objective/kl": 45.49590301513672, + "objective/non_score_reward": -2.2747950553894043, + "objective/rlhf_reward": 13.912704467773438, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010373364202678204, + "policy/clipfrac_avg": 0.006608482450246811, + "policy/entropy_avg": 0.2166609764099121, + "step": 996, + "val/clipfrac_avg": 0.03890983760356903, + "val/num_eos_tokens": 11, + "val/ratio": 0.986150860786438, + "val/ratio_var": 8.553406951250508e-05 + }, + { + "episode": 15952, + "epoch": 0.8056565656565656, + "eps": 0, + "loss/policy_avg": -0.22047999501228333, + "loss/value_avg": 0.5745018720626831, + "lr": 1.0089670656036137e-06, + "objective/entropy": -114.1541976928711, + "objective/kl": 35.899574279785156, + "objective/non_score_reward": -1.7949787378311157, + "objective/rlhf_reward": 14.869083404541016, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007367507787421346, + "policy/clipfrac_avg": 0.0037586798425763845, + "policy/entropy_avg": 0.15613555908203125, + "step": 997, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 560, + "val/ratio": 0.9902479648590088, + "val/ratio_var": 5.6676559324841946e-05 + }, + { + "episode": 15968, + "epoch": 0.8064646464646464, + "eps": 0, + "loss/policy_avg": -0.04391951486468315, + "loss/value_avg": 0.6516869068145752, + "lr": 1.0009352084829871e-06, + "objective/entropy": -112.42098999023438, + "objective/kl": 39.436988830566406, + "objective/non_score_reward": -1.9718495607376099, + "objective/rlhf_reward": 14.48127555847168, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009028672939166427, + "policy/clipfrac_avg": 0.0059207589365541935, + "policy/entropy_avg": 0.18039512634277344, + "step": 998, + "val/clipfrac_avg": 0.0127427838742733, + "val/num_eos_tokens": 515, + "val/ratio": 0.9878066778182983, + "val/ratio_var": 8.427277498412877e-05 + }, + { + "episode": 15984, + "epoch": 0.8072727272727273, + "eps": 0, + "loss/policy_avg": -0.5726251602172852, + "loss/value_avg": 0.6322129964828491, + "lr": 9.929318906602176e-07, + "objective/entropy": -115.74237060546875, + "objective/kl": 35.111915588378906, + "objective/non_score_reward": -1.7555959224700928, + "objective/rlhf_reward": 14.103778839111328, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006586643867194653, + "policy/clipfrac_avg": 0.006481423042714596, + "policy/entropy_avg": 0.1394972801208496, + "step": 999, + "val/clipfrac_avg": 0.019941631704568863, + "val/num_eos_tokens": 81, + "val/ratio": 0.9906902313232422, + "val/ratio_var": 5.2608244004659355e-05 + }, + { + "episode": 16000, + "epoch": 0.8080808080808081, + "eps": 0, + "loss/policy_avg": -0.1571139395236969, + "loss/value_avg": 0.8596901297569275, + "lr": 9.849571692509257e-07, + "objective/entropy": -35.66822052001953, + "objective/kl": 36.119667053222656, + "objective/non_score_reward": -1.805983304977417, + "objective/rlhf_reward": 12.842453956604004, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008497819071635604, + "policy/clipfrac_avg": 0.0041199941188097, + "policy/entropy_avg": 0.16556644439697266, + "step": 1000, + "val/clipfrac_avg": 0.022714650258421898, + "val/num_eos_tokens": 82, + "val/ratio": 0.9879409074783325, + "val/ratio_var": 6.342474807752296e-05 + }, + { + "episode": 16016, + "epoch": 0.8088888888888889, + "eps": 0, + "loss/policy_avg": -0.10610047727823257, + "loss/value_avg": 0.5593327283859253, + "lr": 9.770111011666582e-07, + "objective/entropy": -112.11347961425781, + "objective/kl": 33.91107177734375, + "objective/non_score_reward": -1.6955535411834717, + "objective/rlhf_reward": 13.304447174072266, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008447067812085152, + "policy/clipfrac_avg": 0.004359187558293343, + "policy/entropy_avg": 0.18085873126983643, + "step": 1001, + "val/clipfrac_avg": 0.0036525975447148085, + "val/num_eos_tokens": 37, + "val/ratio": 0.9888993501663208, + "val/ratio_var": 0.00012090770906070247 + }, + { + "episode": 16032, + "epoch": 0.8096969696969697, + "eps": 0, + "loss/policy_avg": -0.4011276960372925, + "loss/value_avg": 0.8269133567810059, + "lr": 9.690937431144725e-07, + "objective/entropy": -143.82159423828125, + "objective/kl": 32.61180877685547, + "objective/non_score_reward": -1.6305904388427734, + "objective/rlhf_reward": 13.322534561157227, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0007577177020721138, + "policy/clipfrac_avg": 0.005468646064400673, + "policy/entropy_avg": 0.14389371871948242, + "step": 1002, + "val/clipfrac_avg": 0.016304347664117813, + "val/num_eos_tokens": 153, + "val/ratio": 0.990355908870697, + "val/ratio_var": 4.9998925533145666e-05 + }, + { + "episode": 16048, + "epoch": 0.8105050505050505, + "eps": 0, + "loss/policy_avg": -0.18046468496322632, + "loss/value_avg": 0.4406319260597229, + "lr": 9.612051515965388e-07, + "objective/entropy": -41.019737243652344, + "objective/kl": 43.84602355957031, + "objective/non_score_reward": -2.1923012733459473, + "objective/rlhf_reward": 14.870198249816895, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009248010464943945, + "policy/clipfrac_avg": 0.004084739368408918, + "policy/entropy_avg": 0.1930408477783203, + "step": 1003, + "val/clipfrac_avg": 0.01253926008939743, + "val/num_eos_tokens": 425, + "val/ratio": 0.9877877235412598, + "val/ratio_var": 4.6758981625316665e-05 + }, + { + "episode": 16064, + "epoch": 0.8113131313131313, + "eps": 0, + "loss/policy_avg": -0.03701537847518921, + "loss/value_avg": 0.4312940835952759, + "lr": 9.533453829097355e-07, + "objective/entropy": -115.09342956542969, + "objective/kl": 43.825538635253906, + "objective/non_score_reward": -2.191277027130127, + "objective/rlhf_reward": 14.941534996032715, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008623325265944004, + "policy/clipfrac_avg": 0.004582014866173267, + "policy/entropy_avg": 0.17541944980621338, + "step": 1004, + "val/clipfrac_avg": 0.02018960565328598, + "val/num_eos_tokens": 259, + "val/ratio": 0.988018274307251, + "val/ratio_var": 5.916683949180879e-05 + }, + { + "episode": 16080, + "epoch": 0.8121212121212121, + "eps": 0, + "loss/policy_avg": 0.30280348658561707, + "loss/value_avg": 0.7898949384689331, + "lr": 9.455144931452459e-07, + "objective/entropy": -62.099971771240234, + "objective/kl": 30.437484741210938, + "objective/non_score_reward": -1.5218743085861206, + "objective/rlhf_reward": 13.712501525878906, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007971136947162449, + "policy/clipfrac_avg": 0.008183586411178112, + "policy/entropy_avg": 0.1460132598876953, + "step": 1005, + "val/clipfrac_avg": 0.003989361692219973, + "val/num_eos_tokens": 215, + "val/ratio": 0.9896875023841858, + "val/ratio_var": 4.330440060584806e-05 + }, + { + "episode": 16096, + "epoch": 0.812929292929293, + "eps": 0, + "loss/policy_avg": -0.23599471151828766, + "loss/value_avg": 1.1437101364135742, + "lr": 9.377125381881607e-07, + "objective/entropy": -46.674827575683594, + "objective/kl": 33.718963623046875, + "objective/non_score_reward": -1.6859480142593384, + "objective/rlhf_reward": 14.86874008178711, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.00085799815133214, + "policy/clipfrac_avg": 0.003210378112271428, + "policy/entropy_avg": 0.19154703617095947, + "step": 1006, + "val/clipfrac_avg": 0.010907451622188091, + "val/num_eos_tokens": 259, + "val/ratio": 0.9871425628662109, + "val/ratio_var": 6.926390778971836e-05 + }, + { + "episode": 16112, + "epoch": 0.8137373737373738, + "eps": 0, + "loss/policy_avg": -0.02023145742714405, + "loss/value_avg": 1.2994359731674194, + "lr": 9.299395737170758e-07, + "objective/entropy": -87.60648345947266, + "objective/kl": 35.53752899169922, + "objective/non_score_reward": -1.7768765687942505, + "objective/rlhf_reward": 15.644998550415039, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007149496814236045, + "policy/clipfrac_avg": 0.002271715085953474, + "policy/entropy_avg": 0.1492328643798828, + "step": 1007, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 12, + "val/ratio": 0.9887728691101074, + "val/ratio_var": 5.388016870710999e-05 + }, + { + "episode": 16128, + "epoch": 0.8145454545454546, + "eps": 0, + "loss/policy_avg": 0.022628188133239746, + "loss/value_avg": 1.1084355115890503, + "lr": 9.221956552036992e-07, + "objective/entropy": -63.064239501953125, + "objective/kl": 36.12407302856445, + "objective/non_score_reward": -1.8062036037445068, + "objective/rlhf_reward": 15.146921157836914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008970820344984531, + "policy/clipfrac_avg": 0.004715266637504101, + "policy/entropy_avg": 0.1754002571105957, + "step": 1008, + "val/clipfrac_avg": 0.008854166604578495, + "val/num_eos_tokens": 540, + "val/ratio": 0.9880603551864624, + "val/ratio_var": 8.214469562517479e-05 + }, + { + "episode": 16144, + "epoch": 0.8153535353535354, + "eps": 0, + "loss/policy_avg": -0.06129603087902069, + "loss/value_avg": 0.558459997177124, + "lr": 9.144808379124509e-07, + "objective/entropy": -104.08320617675781, + "objective/kl": 38.30896759033203, + "objective/non_score_reward": -1.915448546409607, + "objective/rlhf_reward": 13.201739311218262, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008289464749395847, + "policy/clipfrac_avg": 0.0036508054472506046, + "policy/entropy_avg": 0.16502058506011963, + "step": 1009, + "val/clipfrac_avg": 0.013535099104046822, + "val/num_eos_tokens": 360, + "val/ratio": 0.9892135858535767, + "val/ratio_var": 6.506205681944266e-05 + }, + { + "episode": 16160, + "epoch": 0.8161616161616162, + "eps": 0, + "loss/policy_avg": -0.06057693436741829, + "loss/value_avg": 0.288950115442276, + "lr": 9.067951769000699e-07, + "objective/entropy": -71.51614379882812, + "objective/kl": 42.83964538574219, + "objective/non_score_reward": -2.141982078552246, + "objective/rlhf_reward": 14.701767921447754, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009444506140425801, + "policy/clipfrac_avg": 0.002944172825664282, + "policy/entropy_avg": 0.1825122833251953, + "step": 1010, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 613, + "val/ratio": 0.9874241352081299, + "val/ratio_var": 3.9652099076192826e-05 + }, + { + "episode": 16176, + "epoch": 0.816969696969697, + "eps": 0, + "loss/policy_avg": -0.0031431764364242554, + "loss/value_avg": 0.5874146223068237, + "lr": 8.991387270152202e-07, + "objective/entropy": -63.48155212402344, + "objective/kl": 43.46244812011719, + "objective/non_score_reward": -2.1731224060058594, + "objective/rlhf_reward": 14.65500259399414, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009073936380445957, + "policy/clipfrac_avg": 0.005083385389298201, + "policy/entropy_avg": 0.1724100112915039, + "step": 1011, + "val/clipfrac_avg": 0.0003350692568346858, + "val/num_eos_tokens": 564, + "val/ratio": 0.9878172874450684, + "val/ratio_var": 5.2520466852001846e-05 + }, + { + "episode": 16192, + "epoch": 0.8177777777777778, + "eps": 0, + "loss/policy_avg": 0.020213183015584946, + "loss/value_avg": 0.3557816445827484, + "lr": 8.915115428981064e-07, + "objective/entropy": -24.23276138305664, + "objective/kl": 30.50366973876953, + "objective/non_score_reward": -1.5251834392547607, + "objective/rlhf_reward": 14.45919132232666, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009571783011779189, + "policy/clipfrac_avg": 0.004434539936482906, + "policy/entropy_avg": 0.18828392028808594, + "step": 1012, + "val/clipfrac_avg": 0.015398487448692322, + "val/num_eos_tokens": 104, + "val/ratio": 0.9878268241882324, + "val/ratio_var": 6.845602911198512e-05 + }, + { + "episode": 16208, + "epoch": 0.8185858585858586, + "eps": 0, + "loss/policy_avg": -0.06331466138362885, + "loss/value_avg": 0.6016228795051575, + "lr": 8.839136789800729e-07, + "objective/entropy": -72.25382995605469, + "objective/kl": 35.868072509765625, + "objective/non_score_reward": -1.7934037446975708, + "objective/rlhf_reward": 14.183158874511719, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008664189372211695, + "policy/clipfrac_avg": 0.0032933237962424755, + "policy/entropy_avg": 0.17766380310058594, + "step": 1013, + "val/clipfrac_avg": 0.006926534231752157, + "val/num_eos_tokens": 294, + "val/ratio": 0.987805962562561, + "val/ratio_var": 6.278733781073242e-05 + }, + { + "episode": 16224, + "epoch": 0.8193939393939393, + "eps": 0, + "loss/policy_avg": 0.06992897391319275, + "loss/value_avg": 0.47664809226989746, + "lr": 8.76345189483222e-07, + "objective/entropy": -88.55311584472656, + "objective/kl": 30.213661193847656, + "objective/non_score_reward": -1.5106831789016724, + "objective/rlhf_reward": 13.747129440307617, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007972473395057023, + "policy/clipfrac_avg": 0.00806876365095377, + "policy/entropy_avg": 0.1417522430419922, + "step": 1014, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 51, + "val/ratio": 0.9906024932861328, + "val/ratio_var": 5.377350316848606e-05 + }, + { + "episode": 16240, + "epoch": 0.8202020202020202, + "eps": 0, + "loss/policy_avg": 0.11788127571344376, + "loss/value_avg": 0.528445839881897, + "lr": 8.688061284200266e-07, + "objective/entropy": -92.63037109375, + "objective/kl": 36.9803466796875, + "objective/non_score_reward": -1.84901762008667, + "objective/rlhf_reward": 14.065045356750488, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008528477046638727, + "policy/clipfrac_avg": 0.0033233293797820807, + "policy/entropy_avg": 0.17418956756591797, + "step": 1015, + "val/clipfrac_avg": 0.013305861502885818, + "val/num_eos_tokens": 387, + "val/ratio": 0.989635169506073, + "val/ratio_var": 4.9907346692634746e-05 + }, + { + "episode": 16256, + "epoch": 0.821010101010101, + "eps": 0, + "loss/policy_avg": -0.04949585720896721, + "loss/value_avg": 0.3462344706058502, + "lr": 8.61296549592941e-07, + "objective/entropy": -62.09973907470703, + "objective/kl": 44.77581024169922, + "objective/non_score_reward": -2.23879075050354, + "objective/rlhf_reward": 12.683084487915039, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.001046380726620555, + "policy/clipfrac_avg": 0.00542850699275732, + "policy/entropy_avg": 0.2170124053955078, + "step": 1016, + "val/clipfrac_avg": 0.03059275820851326, + "val/num_eos_tokens": 270, + "val/ratio": 0.9858708381652832, + "val/ratio_var": 6.539487367263064e-05 + }, + { + "episode": 16272, + "epoch": 0.8218181818181818, + "eps": 0, + "loss/policy_avg": 0.03657815232872963, + "loss/value_avg": 0.687595546245575, + "lr": 8.538165065940263e-07, + "objective/entropy": -65.8448486328125, + "objective/kl": 34.462608337402344, + "objective/non_score_reward": -1.7231305837631226, + "objective/rlhf_reward": 13.925307273864746, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000925581669434905, + "policy/clipfrac_avg": 0.00542820617556572, + "policy/entropy_avg": 0.19491100311279297, + "step": 1017, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 15, + "val/ratio": 0.9877764582633972, + "val/ratio_var": 6.733841291861609e-05 + }, + { + "episode": 16288, + "epoch": 0.8226262626262626, + "eps": 0, + "loss/policy_avg": -0.097564198076725, + "loss/value_avg": 0.5997859835624695, + "lr": 8.463660528045514e-07, + "objective/entropy": -21.665847778320312, + "objective/kl": 41.799583435058594, + "objective/non_score_reward": -2.0899791717529297, + "objective/rlhf_reward": 14.79283332824707, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010365031193941832, + "policy/clipfrac_avg": 0.00511179631575942, + "policy/entropy_avg": 0.20884323120117188, + "step": 1018, + "val/clipfrac_avg": 0.008413461968302727, + "val/num_eos_tokens": 123, + "val/ratio": 0.9861449003219604, + "val/ratio_var": 5.511847120942548e-05 + }, + { + "episode": 16304, + "epoch": 0.8234343434343434, + "eps": 0, + "loss/policy_avg": -0.06873838603496552, + "loss/value_avg": 0.2803807854652405, + "lr": 8.389452413946314e-07, + "objective/entropy": -106.4912109375, + "objective/kl": 40.70600891113281, + "objective/non_score_reward": -2.0353002548217773, + "objective/rlhf_reward": 13.745949745178223, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008316466701216996, + "policy/clipfrac_avg": 0.0057511827908456326, + "policy/entropy_avg": 0.1492767333984375, + "step": 1019, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 205, + "val/ratio": 0.9896730184555054, + "val/ratio_var": 4.522735616774298e-05 + }, + { + "episode": 16320, + "epoch": 0.8242424242424242, + "eps": 0, + "loss/policy_avg": 0.018638040870428085, + "loss/value_avg": 0.7235357761383057, + "lr": 8.315541253228332e-07, + "objective/entropy": -29.912065505981445, + "objective/kl": 32.72511291503906, + "objective/non_score_reward": -1.6362557411193848, + "objective/rlhf_reward": 13.707494735717773, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0009581202175468206, + "policy/clipfrac_avg": 0.0044068810530006886, + "policy/entropy_avg": 0.2027587890625, + "step": 1020, + "val/clipfrac_avg": 0.009765625, + "val/num_eos_tokens": 35, + "val/ratio": 0.9886679649353027, + "val/ratio_var": 7.175742939580232e-05 + }, + { + "episode": 16336, + "epoch": 0.825050505050505, + "eps": 0, + "loss/policy_avg": 0.15834374725818634, + "loss/value_avg": 0.9719556570053101, + "lr": 8.241927573358078e-07, + "objective/entropy": -17.822988510131836, + "objective/kl": 44.23792266845703, + "objective/non_score_reward": -2.2118961811065674, + "objective/rlhf_reward": 13.405291557312012, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000986535451374948, + "policy/clipfrac_avg": 0.0043609049171209335, + "policy/entropy_avg": 0.19144058227539062, + "step": 1021, + "val/clipfrac_avg": 0.009959635324776173, + "val/num_eos_tokens": 44, + "val/ratio": 0.9864338040351868, + "val/ratio_var": 2.8990581995458342e-05 + }, + { + "episode": 16352, + "epoch": 0.8258585858585858, + "eps": 0, + "loss/policy_avg": 0.08962170779705048, + "loss/value_avg": 0.22370219230651855, + "lr": 8.168611899679013e-07, + "objective/entropy": -87.69083404541016, + "objective/kl": 41.61705017089844, + "objective/non_score_reward": -2.080852508544922, + "objective/rlhf_reward": 14.262897491455078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008991376380436122, + "policy/clipfrac_avg": 0.004512319341301918, + "policy/entropy_avg": 0.190032958984375, + "step": 1022, + "val/clipfrac_avg": 0.04466811940073967, + "val/num_eos_tokens": 17, + "val/ratio": 0.9874501824378967, + "val/ratio_var": 6.216822657734156e-05 + }, + { + "episode": 16368, + "epoch": 0.8266666666666667, + "eps": 0, + "loss/policy_avg": -0.6490851044654846, + "loss/value_avg": 0.6145610213279724, + "lr": 8.095594755407971e-07, + "objective/entropy": -103.04541778564453, + "objective/kl": 37.85612487792969, + "objective/non_score_reward": -1.8928062915802002, + "objective/rlhf_reward": 14.255630493164062, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009121837792918086, + "policy/clipfrac_avg": 0.004552682861685753, + "policy/entropy_avg": 0.17924374341964722, + "step": 1023, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 624, + "val/ratio": 0.9885929822921753, + "val/ratio_var": 8.515298395650461e-05 + }, + { + "episode": 16384, + "epoch": 0.8274747474747475, + "eps": 0, + "loss/policy_avg": 0.09050679206848145, + "loss/value_avg": 1.2950150966644287, + "lr": 8.022876661631268e-07, + "objective/entropy": -46.773006439208984, + "objective/kl": 33.627471923828125, + "objective/non_score_reward": -1.6813737154006958, + "objective/rlhf_reward": 14.521751403808594, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009958728915080428, + "policy/clipfrac_avg": 0.0035771888215094805, + "policy/entropy_avg": 0.17354488372802734, + "step": 1024, + "val/clipfrac_avg": 0.0034391595982015133, + "val/num_eos_tokens": 5, + "val/ratio": 0.9874868392944336, + "val/ratio_var": 7.542264211224392e-05 + }, + { + "episode": 16400, + "epoch": 0.8282828282828283, + "eps": 0, + "loss/policy_avg": 0.09637929499149323, + "loss/value_avg": 0.3471735417842865, + "lr": 7.950458137301082e-07, + "objective/entropy": -118.60977172851562, + "objective/kl": 40.922630310058594, + "objective/non_score_reward": -2.0461313724517822, + "objective/rlhf_reward": 14.555431365966797, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007377723813988268, + "policy/clipfrac_avg": 0.004692882765084505, + "policy/entropy_avg": 0.1540050506591797, + "step": 1025, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 445, + "val/ratio": 0.9898372888565063, + "val/ratio_var": 4.050094139529392e-05 + }, + { + "episode": 16416, + "epoch": 0.8290909090909091, + "eps": 0, + "loss/policy_avg": 0.003938224166631699, + "loss/value_avg": 0.12186452746391296, + "lr": 7.878339699231702e-07, + "objective/entropy": 5.97334623336792, + "objective/kl": 47.65478515625, + "objective/non_score_reward": -2.382739543914795, + "objective/rlhf_reward": 15.882884979248047, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011957846581935883, + "policy/clipfrac_avg": 0.0038270659279078245, + "policy/entropy_avg": 0.24349212646484375, + "step": 1026, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 63, + "val/ratio": 0.9836880564689636, + "val/ratio_var": 3.132951314910315e-05 + }, + { + "episode": 16432, + "epoch": 0.8298989898989899, + "eps": 0, + "loss/policy_avg": -0.008172400295734406, + "loss/value_avg": 0.40194565057754517, + "lr": 7.806521862095834e-07, + "objective/entropy": -145.87228393554688, + "objective/kl": 43.73109817504883, + "objective/non_score_reward": -2.1865549087524414, + "objective/rlhf_reward": 12.707976341247559, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000986664555966854, + "policy/clipfrac_avg": 0.005425687879323959, + "policy/entropy_avg": 0.21741676330566406, + "step": 1027, + "val/clipfrac_avg": 0.00168609619140625, + "val/num_eos_tokens": 2002, + "val/ratio": 0.9879361987113953, + "val/ratio_var": 7.765040209051222e-05 + }, + { + "episode": 16448, + "epoch": 0.8307070707070707, + "eps": 0, + "loss/policy_avg": -0.040230363607406616, + "loss/value_avg": 0.2879355549812317, + "lr": 7.735005138421003e-07, + "objective/entropy": -39.62914276123047, + "objective/kl": 30.83833122253418, + "objective/non_score_reward": -1.5419166088104248, + "objective/rlhf_reward": 13.919020652770996, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008982427534647286, + "policy/clipfrac_avg": 0.004154416732490063, + "policy/entropy_avg": 0.1513214111328125, + "step": 1028, + "val/clipfrac_avg": 0.0003633720916695893, + "val/num_eos_tokens": 42, + "val/ratio": 0.9900394678115845, + "val/ratio_var": 4.5976830733707175e-05 + }, + { + "episode": 16464, + "epoch": 0.8315151515151515, + "eps": 0, + "loss/policy_avg": -0.17712682485580444, + "loss/value_avg": 0.41484764218330383, + "lr": 7.663790038585794e-07, + "objective/entropy": -91.81086730957031, + "objective/kl": 39.669403076171875, + "objective/non_score_reward": -1.9834702014923096, + "objective/rlhf_reward": 14.73528003692627, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009472984238527715, + "policy/clipfrac_avg": 0.0033632363192737103, + "policy/entropy_avg": 0.1894826889038086, + "step": 1029, + "val/clipfrac_avg": 0.015109826810657978, + "val/num_eos_tokens": 637, + "val/ratio": 0.9873372316360474, + "val/ratio_var": 6.729063170496374e-05 + }, + { + "episode": 16480, + "epoch": 0.8323232323232324, + "eps": 0, + "loss/policy_avg": 0.03287976607680321, + "loss/value_avg": 0.3657122254371643, + "lr": 7.592877070816268e-07, + "objective/entropy": -81.92291259765625, + "objective/kl": 40.63905334472656, + "objective/non_score_reward": -2.0319526195526123, + "objective/rlhf_reward": 14.960234642028809, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009109035599976778, + "policy/clipfrac_avg": 0.003938743844628334, + "policy/entropy_avg": 0.17749857902526855, + "step": 1030, + "val/clipfrac_avg": 0.005359000526368618, + "val/num_eos_tokens": 56, + "val/ratio": 0.9891733527183533, + "val/ratio_var": 5.9037287428509444e-05 + }, + { + "episode": 16496, + "epoch": 0.8331313131313132, + "eps": 0, + "loss/policy_avg": 0.054874081164598465, + "loss/value_avg": 0.28462013602256775, + "lr": 7.522266741182305e-07, + "objective/entropy": -79.7171630859375, + "objective/kl": 43.41844177246094, + "objective/non_score_reward": -2.170922040939331, + "objective/rlhf_reward": 13.88376522064209, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009897679556161165, + "policy/clipfrac_avg": 0.00460906233638525, + "policy/entropy_avg": 0.18915176391601562, + "step": 1031, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 413, + "val/ratio": 0.987737238407135, + "val/ratio_var": 6.536820001201704e-05 + }, + { + "episode": 16512, + "epoch": 0.833939393939394, + "eps": 0, + "loss/policy_avg": 0.42096102237701416, + "loss/value_avg": 0.7095996141433716, + "lr": 7.451959553594051e-07, + "objective/entropy": -75.63118743896484, + "objective/kl": 32.546966552734375, + "objective/non_score_reward": -1.6273484230041504, + "objective/rlhf_reward": 12.075777053833008, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.000955111114308238, + "policy/clipfrac_avg": 0.005775670520961285, + "policy/entropy_avg": 0.16732215881347656, + "step": 1032, + "val/clipfrac_avg": 0.017921777442097664, + "val/num_eos_tokens": 145, + "val/ratio": 0.989020586013794, + "val/ratio_var": 4.018365507363342e-05 + }, + { + "episode": 16528, + "epoch": 0.8347474747474748, + "eps": 0, + "loss/policy_avg": 0.017501141875982285, + "loss/value_avg": 0.4045110046863556, + "lr": 7.381956009798253e-07, + "objective/entropy": -53.61748123168945, + "objective/kl": 38.599403381347656, + "objective/non_score_reward": -1.929970145225525, + "objective/rlhf_reward": 12.640341758728027, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008390340954065323, + "policy/clipfrac_avg": 0.0041229561902582645, + "policy/entropy_avg": 0.18261051177978516, + "step": 1033, + "val/clipfrac_avg": 0.020542385056614876, + "val/num_eos_tokens": 139, + "val/ratio": 0.988516092300415, + "val/ratio_var": 9.491335367783904e-05 + }, + { + "episode": 16544, + "epoch": 0.8355555555555556, + "eps": 0, + "loss/policy_avg": -0.03480757772922516, + "loss/value_avg": 1.401247501373291, + "lr": 7.312256609374708e-07, + "objective/entropy": -103.81524658203125, + "objective/kl": 36.218544006347656, + "objective/non_score_reward": -1.810927152633667, + "objective/rlhf_reward": 14.07969856262207, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007683830335736275, + "policy/clipfrac_avg": 0.003070900682359934, + "policy/entropy_avg": 0.14527511596679688, + "step": 1034, + "val/clipfrac_avg": 0.03183094039559364, + "val/num_eos_tokens": 65, + "val/ratio": 0.9899626970291138, + "val/ratio_var": 6.579139153473079e-05 + }, + { + "episode": 16560, + "epoch": 0.8363636363636363, + "eps": 0, + "loss/policy_avg": -0.28031888604164124, + "loss/value_avg": 1.1235523223876953, + "lr": 7.242861849732696e-07, + "objective/entropy": -59.279266357421875, + "objective/kl": 38.71485900878906, + "objective/non_score_reward": -1.935742974281311, + "objective/rlhf_reward": 14.07206916809082, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007580246892757714, + "policy/clipfrac_avg": 0.003035132307559252, + "policy/entropy_avg": 0.14683592319488525, + "step": 1035, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 109, + "val/ratio": 0.9899235963821411, + "val/ratio_var": 5.906899241381325e-05 + }, + { + "episode": 16576, + "epoch": 0.8371717171717171, + "eps": 0, + "loss/policy_avg": -0.14042839407920837, + "loss/value_avg": 1.1183263063430786, + "lr": 7.173772226107434e-07, + "objective/entropy": -64.92012023925781, + "objective/kl": 38.85520553588867, + "objective/non_score_reward": -1.9427603483200073, + "objective/rlhf_reward": 14.400989532470703, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009576823795214295, + "policy/clipfrac_avg": 0.004880891181528568, + "policy/entropy_avg": 0.19240808486938477, + "step": 1036, + "val/clipfrac_avg": 0.012853971682488918, + "val/num_eos_tokens": 155, + "val/ratio": 0.987326979637146, + "val/ratio_var": 9.263003448722884e-05 + }, + { + "episode": 16592, + "epoch": 0.8379797979797979, + "eps": 0, + "loss/policy_avg": -0.03311082720756531, + "loss/value_avg": 0.3677228093147278, + "lr": 7.104988231556543e-07, + "objective/entropy": -48.31360626220703, + "objective/kl": 36.47294616699219, + "objective/non_score_reward": -1.8236472606658936, + "objective/rlhf_reward": 14.137290000915527, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008876229985617101, + "policy/clipfrac_avg": 0.002901179250329733, + "policy/entropy_avg": 0.20080184936523438, + "step": 1037, + "val/clipfrac_avg": 0.002261103130877018, + "val/num_eos_tokens": 20, + "val/ratio": 0.9893977642059326, + "val/ratio_var": 5.7600576838012785e-05 + }, + { + "episode": 16608, + "epoch": 0.8387878787878787, + "eps": 0, + "loss/policy_avg": -0.10436652600765228, + "loss/value_avg": 0.43171289563179016, + "lr": 7.036510356956494e-07, + "objective/entropy": -66.36492919921875, + "objective/kl": 41.474632263183594, + "objective/non_score_reward": -2.0737314224243164, + "objective/rlhf_reward": 15.168456077575684, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009756431682035327, + "policy/clipfrac_avg": 0.005543471779674292, + "policy/entropy_avg": 0.188018798828125, + "step": 1038, + "val/clipfrac_avg": 0.01755008101463318, + "val/num_eos_tokens": 3, + "val/ratio": 0.9874497652053833, + "val/ratio_var": 5.146233888808638e-05 + }, + { + "episode": 16624, + "epoch": 0.8395959595959596, + "eps": 0, + "loss/policy_avg": -0.23364350199699402, + "loss/value_avg": 0.3087168335914612, + "lr": 6.968339090999188e-07, + "objective/entropy": -82.42080688476562, + "objective/kl": 33.46930694580078, + "objective/non_score_reward": -1.6734652519226074, + "objective/rlhf_reward": 16.849971771240234, + "objective/scores": 18.5, + "policy/approxkl_avg": 0.0009166139643639326, + "policy/clipfrac_avg": 0.0028331982903182507, + "policy/entropy_avg": 0.18764019012451172, + "step": 1039, + "val/clipfrac_avg": 0.022823384031653404, + "val/num_eos_tokens": 150, + "val/ratio": 0.9876014590263367, + "val/ratio_var": 7.868735701777041e-05 + }, + { + "episode": 16640, + "epoch": 0.8404040404040404, + "eps": 0, + "loss/policy_avg": -0.24348971247673035, + "loss/value_avg": 0.46729177236557007, + "lr": 6.900474920188383e-07, + "objective/entropy": -98.47803497314453, + "objective/kl": 31.56005096435547, + "objective/non_score_reward": -1.5780024528503418, + "objective/rlhf_reward": 12.9376220703125, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0007581596728414297, + "policy/clipfrac_avg": 0.0029252313543111086, + "policy/entropy_avg": 0.14857721328735352, + "step": 1040, + "val/clipfrac_avg": 0.018430940806865692, + "val/num_eos_tokens": 64, + "val/ratio": 0.9900062084197998, + "val/ratio_var": 7.637133239768445e-05 + }, + { + "episode": 16656, + "epoch": 0.8412121212121212, + "eps": 0, + "loss/policy_avg": -0.3006308078765869, + "loss/value_avg": 0.9436385035514832, + "lr": 6.832918328836247e-07, + "objective/entropy": -67.75382995605469, + "objective/kl": 32.892269134521484, + "objective/non_score_reward": -1.64461350440979, + "objective/rlhf_reward": 14.644449234008789, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009258019272238016, + "policy/clipfrac_avg": 0.004761096090078354, + "policy/entropy_avg": 0.1688389778137207, + "step": 1041, + "val/clipfrac_avg": 0.018809743225574493, + "val/num_eos_tokens": 357, + "val/ratio": 0.9893865585327148, + "val/ratio_var": 6.368513277266175e-05 + }, + { + "episode": 16672, + "epoch": 0.842020202020202, + "eps": 0, + "loss/policy_avg": -0.06437136977910995, + "loss/value_avg": 0.4288250207901001, + "lr": 6.765669799059915e-07, + "objective/entropy": -74.0018310546875, + "objective/kl": 35.5787467956543, + "objective/non_score_reward": -1.7789373397827148, + "objective/rlhf_reward": 13.291374206542969, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008827393758110702, + "policy/clipfrac_avg": 0.004510640166699886, + "policy/entropy_avg": 0.174835205078125, + "step": 1042, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 88, + "val/ratio": 0.9889439344406128, + "val/ratio_var": 5.196507117943838e-05 + }, + { + "episode": 16688, + "epoch": 0.8428282828282828, + "eps": 0, + "loss/policy_avg": 0.08304592221975327, + "loss/value_avg": 0.627299427986145, + "lr": 6.698729810778065e-07, + "objective/entropy": -94.1718521118164, + "objective/kl": 36.0411491394043, + "objective/non_score_reward": -1.8020575046539307, + "objective/rlhf_reward": 14.455755233764648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007288270280696452, + "policy/clipfrac_avg": 0.003802281804382801, + "policy/entropy_avg": 0.14253592491149902, + "step": 1043, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 492, + "val/ratio": 0.9900709390640259, + "val/ratio_var": 3.72955619241111e-05 + }, + { + "episode": 16704, + "epoch": 0.8436363636363636, + "eps": 0, + "loss/policy_avg": 0.289115309715271, + "loss/value_avg": 0.6000362038612366, + "lr": 6.632098841707458e-07, + "objective/entropy": -44.56480407714844, + "objective/kl": 37.80200958251953, + "objective/non_score_reward": -1.8901004791259766, + "objective/rlhf_reward": 13.203649520874023, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0012037131236866117, + "policy/clipfrac_avg": 0.005290274508297443, + "policy/entropy_avg": 0.21977996826171875, + "step": 1044, + "val/clipfrac_avg": 0.0035122863482683897, + "val/num_eos_tokens": 378, + "val/ratio": 0.9847176671028137, + "val/ratio_var": 9.624927770346403e-05 + }, + { + "episode": 16720, + "epoch": 0.8444444444444444, + "eps": 0, + "loss/policy_avg": 0.07545819133520126, + "loss/value_avg": 0.46948039531707764, + "lr": 6.565777367359516e-07, + "objective/entropy": -68.83721923828125, + "objective/kl": 39.69865798950195, + "objective/non_score_reward": -1.9849328994750977, + "objective/rlhf_reward": 13.468192100524902, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008786184480413795, + "policy/clipfrac_avg": 0.004983194172382355, + "policy/entropy_avg": 0.17647552490234375, + "step": 1045, + "val/clipfrac_avg": 0.00048828125, + "val/num_eos_tokens": 99, + "val/ratio": 0.9880126714706421, + "val/ratio_var": 4.486176840146072e-05 + }, + { + "episode": 16736, + "epoch": 0.8452525252525253, + "eps": 0, + "loss/policy_avg": 0.18258720636367798, + "loss/value_avg": 0.33022794127464294, + "lr": 6.499765861036994e-07, + "objective/entropy": -133.97775268554688, + "objective/kl": 39.10038375854492, + "objective/non_score_reward": -1.9550191164016724, + "objective/rlhf_reward": 13.568418502807617, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007223144639283419, + "policy/clipfrac_avg": 0.0023505911231040955, + "policy/entropy_avg": 0.1742839813232422, + "step": 1046, + "val/clipfrac_avg": 0.001006610575132072, + "val/num_eos_tokens": 767, + "val/ratio": 0.9903188943862915, + "val/ratio_var": 4.2478550312807783e-05 + }, + { + "episode": 16752, + "epoch": 0.8460606060606061, + "eps": 0, + "loss/policy_avg": 0.004041165113449097, + "loss/value_avg": 0.2533167600631714, + "lr": 6.43406479383053e-07, + "objective/entropy": -11.881628036499023, + "objective/kl": 40.72428894042969, + "objective/non_score_reward": -2.036214590072632, + "objective/rlhf_reward": 15.112222671508789, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0010564490221440792, + "policy/clipfrac_avg": 0.007933112792670727, + "policy/entropy_avg": 0.20357513427734375, + "step": 1047, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.9868043661117554, + "val/ratio_var": 6.374649819917977e-05 + }, + { + "episode": 16768, + "epoch": 0.8468686868686869, + "eps": 0, + "loss/policy_avg": 0.0763624757528305, + "loss/value_avg": 0.4388294219970703, + "lr": 6.368674634615357e-07, + "objective/entropy": -45.377681732177734, + "objective/kl": 34.73613739013672, + "objective/non_score_reward": -1.736806869506836, + "objective/rlhf_reward": 14.192880630493164, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009812908247113228, + "policy/clipfrac_avg": 0.0049714683555066586, + "policy/entropy_avg": 0.18614768981933594, + "step": 1048, + "val/clipfrac_avg": 0.0006127451197244227, + "val/num_eos_tokens": 351, + "val/ratio": 0.98753422498703, + "val/ratio_var": 4.261466165189631e-05 + }, + { + "episode": 16784, + "epoch": 0.8476767676767677, + "eps": 0, + "loss/policy_avg": -0.12992079555988312, + "loss/value_avg": 0.939970850944519, + "lr": 6.303595850047844e-07, + "objective/entropy": -80.03951263427734, + "objective/kl": 42.18169021606445, + "objective/non_score_reward": -2.1090846061706543, + "objective/rlhf_reward": 13.812790870666504, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009452086524106562, + "policy/clipfrac_avg": 0.004814188927412033, + "policy/entropy_avg": 0.1846628189086914, + "step": 1049, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 193, + "val/ratio": 0.988731324672699, + "val/ratio_var": 6.252430466702208e-05 + }, + { + "episode": 16800, + "epoch": 0.8484848484848485, + "eps": 0, + "loss/policy_avg": -0.12597322463989258, + "loss/value_avg": 0.5964376926422119, + "lr": 6.238828904562316e-07, + "objective/entropy": -56.887001037597656, + "objective/kl": 37.25836944580078, + "objective/non_score_reward": -1.8629186153411865, + "objective/rlhf_reward": 14.301143646240234, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008069226169027388, + "policy/clipfrac_avg": 0.0021998940501362085, + "policy/entropy_avg": 0.17313003540039062, + "step": 1050, + "val/clipfrac_avg": 0.00014467592700384557, + "val/num_eos_tokens": 69, + "val/ratio": 0.9889218807220459, + "val/ratio_var": 4.452227949514054e-05 + }, + { + "episode": 16816, + "epoch": 0.8492929292929293, + "eps": 0, + "loss/policy_avg": -0.284543514251709, + "loss/value_avg": 0.4945632815361023, + "lr": 6.174374260367611e-07, + "objective/entropy": -162.38363647460938, + "objective/kl": 36.81440734863281, + "objective/non_score_reward": -1.8407204151153564, + "objective/rlhf_reward": 13.549904823303223, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0006882078596390784, + "policy/clipfrac_avg": 0.0029306234791874886, + "policy/entropy_avg": 0.13589084148406982, + "step": 1051, + "val/clipfrac_avg": 0.02376413717865944, + "val/num_eos_tokens": 810, + "val/ratio": 0.9912172555923462, + "val/ratio_var": 4.5012933696853e-05 + }, + { + "episode": 16832, + "epoch": 0.8501010101010101, + "eps": 0, + "loss/policy_avg": -0.011637904681265354, + "loss/value_avg": 0.556053638458252, + "lr": 6.110232377443864e-07, + "objective/entropy": -106.74056243896484, + "objective/kl": 35.95270538330078, + "objective/non_score_reward": -1.7976351976394653, + "objective/rlhf_reward": 16.46017837524414, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0009089771774597466, + "policy/clipfrac_avg": 0.005288749001920223, + "policy/entropy_avg": 0.18283462524414062, + "step": 1052, + "val/clipfrac_avg": 0.0026041667442768812, + "val/num_eos_tokens": 104, + "val/ratio": 0.9882605075836182, + "val/ratio_var": 7.140174420783296e-05 + }, + { + "episode": 16848, + "epoch": 0.850909090909091, + "eps": 0, + "loss/policy_avg": -0.09856721758842468, + "loss/value_avg": 0.43345025181770325, + "lr": 6.04640371353914e-07, + "objective/entropy": -75.6595458984375, + "objective/kl": 40.826602935791016, + "objective/non_score_reward": -2.041330337524414, + "objective/rlhf_reward": 13.857107162475586, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009147390373982489, + "policy/clipfrac_avg": 0.0046502938494086266, + "policy/entropy_avg": 0.21721649169921875, + "step": 1053, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 61, + "val/ratio": 0.9873005151748657, + "val/ratio_var": 7.706157339271158e-05 + }, + { + "episode": 16864, + "epoch": 0.8517171717171718, + "eps": 0, + "loss/policy_avg": 0.051140159368515015, + "loss/value_avg": 0.46211516857147217, + "lr": 5.98288872416628e-07, + "objective/entropy": -99.58453369140625, + "objective/kl": 38.763389587402344, + "objective/non_score_reward": -1.938169240951538, + "objective/rlhf_reward": 15.054018020629883, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008463884005323052, + "policy/clipfrac_avg": 0.0038104294799268246, + "policy/entropy_avg": 0.16744399070739746, + "step": 1054, + "val/clipfrac_avg": 0.0019132653251290321, + "val/num_eos_tokens": 520, + "val/ratio": 0.989348292350769, + "val/ratio_var": 8.285223157145083e-05 + }, + { + "episode": 16880, + "epoch": 0.8525252525252526, + "eps": 0, + "loss/policy_avg": -0.03345664590597153, + "loss/value_avg": 0.41552969813346863, + "lr": 5.919687862599549e-07, + "objective/entropy": -79.0547103881836, + "objective/kl": 37.5797004699707, + "objective/non_score_reward": -1.878985047340393, + "objective/rlhf_reward": 14.566328048706055, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009661274962127209, + "policy/clipfrac_avg": 0.00524417357519269, + "policy/entropy_avg": 0.186629980802536, + "step": 1055, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 305, + "val/ratio": 0.9881995916366577, + "val/ratio_var": 7.692445797147229e-05 + }, + { + "episode": 16896, + "epoch": 0.8533333333333334, + "eps": 0, + "loss/policy_avg": 0.1846567541360855, + "loss/value_avg": 0.5820722579956055, + "lr": 5.856801579871457e-07, + "objective/entropy": -91.50765991210938, + "objective/kl": 38.447975158691406, + "objective/non_score_reward": -1.9223988056182861, + "objective/rlhf_reward": 12.522913932800293, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0007886831881478429, + "policy/clipfrac_avg": 0.00426090694963932, + "policy/entropy_avg": 0.15495681762695312, + "step": 1056, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 137, + "val/ratio": 0.9909533262252808, + "val/ratio_var": 5.192918251850642e-05 + }, + { + "episode": 16912, + "epoch": 0.8541414141414141, + "eps": 0, + "loss/policy_avg": -0.014410331845283508, + "loss/value_avg": 0.45579588413238525, + "lr": 5.794230324769518e-07, + "objective/entropy": -45.45872497558594, + "objective/kl": 41.44641876220703, + "objective/non_score_reward": -2.0723209381103516, + "objective/rlhf_reward": 13.966741561889648, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009499872685410082, + "policy/clipfrac_avg": 0.007878290489315987, + "policy/entropy_avg": 0.18343734741210938, + "step": 1057, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 399, + "val/ratio": 0.9887840747833252, + "val/ratio_var": 6.99822703609243e-05 + }, + { + "episode": 16928, + "epoch": 0.8549494949494949, + "eps": 0, + "loss/policy_avg": -0.05935902148485184, + "loss/value_avg": 0.36739227175712585, + "lr": 5.731974543833041e-07, + "objective/entropy": -55.36639404296875, + "objective/kl": 38.324073791503906, + "objective/non_score_reward": -1.916203498840332, + "objective/rlhf_reward": 12.029109001159668, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009155046427622437, + "policy/clipfrac_avg": 0.003906010650098324, + "policy/entropy_avg": 0.17214488983154297, + "step": 1058, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891675114631653, + "val/ratio_var": 5.269415851216763e-05 + }, + { + "episode": 16944, + "epoch": 0.8557575757575757, + "eps": 0, + "loss/policy_avg": 0.12509797513484955, + "loss/value_avg": 0.5901766419410706, + "lr": 5.670034681349995e-07, + "objective/entropy": -77.39466857910156, + "objective/kl": 39.27944564819336, + "objective/non_score_reward": -1.9639724493026733, + "objective/rlhf_reward": 12.512590408325195, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0011488801101222634, + "policy/clipfrac_avg": 0.005929230712354183, + "policy/entropy_avg": 0.22491836547851562, + "step": 1059, + "val/clipfrac_avg": 0.01009130198508501, + "val/num_eos_tokens": 274, + "val/ratio": 0.9847910404205322, + "val/ratio_var": 8.84500186657533e-05 + }, + { + "episode": 16960, + "epoch": 0.8565656565656565, + "eps": 0, + "loss/policy_avg": 0.027433840557932854, + "loss/value_avg": 0.2936461567878723, + "lr": 5.608411179353767e-07, + "objective/entropy": -68.70003509521484, + "objective/kl": 34.862098693847656, + "objective/non_score_reward": -1.7431049346923828, + "objective/rlhf_reward": 15.538145065307617, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010132534662261605, + "policy/clipfrac_avg": 0.00747870746999979, + "policy/entropy_avg": 0.19480419158935547, + "step": 1060, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 6, + "val/ratio": 0.9880098700523376, + "val/ratio_var": 6.574365397682413e-05 + }, + { + "episode": 16976, + "epoch": 0.8573737373737373, + "eps": 0, + "loss/policy_avg": 0.22168952226638794, + "loss/value_avg": 0.6544172763824463, + "lr": 5.547104477620041e-07, + "objective/entropy": -101.6231918334961, + "objective/kl": 36.893959045410156, + "objective/non_score_reward": -1.844698190689087, + "objective/rlhf_reward": 13.850614547729492, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008218783186748624, + "policy/clipfrac_avg": 0.005969468504190445, + "policy/entropy_avg": 0.15513205528259277, + "step": 1061, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 465, + "val/ratio": 0.9892902374267578, + "val/ratio_var": 6.282331742113456e-05 + }, + { + "episode": 16992, + "epoch": 0.8581818181818182, + "eps": 0, + "loss/policy_avg": -0.23859268426895142, + "loss/value_avg": 0.5853025913238525, + "lr": 5.486115013663668e-07, + "objective/entropy": -112.72584533691406, + "objective/kl": 33.110145568847656, + "objective/non_score_reward": -1.6555070877075195, + "objective/rlhf_reward": 15.28980541229248, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0006752893095836043, + "policy/clipfrac_avg": 0.0032210415229201317, + "policy/entropy_avg": 0.12448978424072266, + "step": 1062, + "val/clipfrac_avg": 4.234417428961024e-05, + "val/num_eos_tokens": 451, + "val/ratio": 0.9915745258331299, + "val/ratio_var": 5.202127795200795e-05 + }, + { + "episode": 17008, + "epoch": 0.858989898989899, + "eps": 0, + "loss/policy_avg": -0.1410359889268875, + "loss/value_avg": 0.22414374351501465, + "lr": 5.425443222735527e-07, + "objective/entropy": -49.841190338134766, + "objective/kl": 38.634700775146484, + "objective/non_score_reward": -1.9317350387573242, + "objective/rlhf_reward": 16.25576400756836, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011730990372598171, + "policy/clipfrac_avg": 0.008051658049225807, + "policy/entropy_avg": 0.22179222106933594, + "step": 1063, + "val/clipfrac_avg": 0.009043844416737556, + "val/num_eos_tokens": 125, + "val/ratio": 0.9854654669761658, + "val/ratio_var": 7.81394264777191e-05 + }, + { + "episode": 17024, + "epoch": 0.8597979797979798, + "eps": 0, + "loss/policy_avg": -0.060928136110305786, + "loss/value_avg": 0.45962244272232056, + "lr": 5.365089537819435e-07, + "objective/entropy": -68.24198913574219, + "objective/kl": 37.95829772949219, + "objective/non_score_reward": -1.8979148864746094, + "objective/rlhf_reward": 15.38333511352539, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008637227001599967, + "policy/clipfrac_avg": 0.006955782417207956, + "policy/entropy_avg": 0.1758570671081543, + "step": 1064, + "val/clipfrac_avg": 0.027727147564291954, + "val/num_eos_tokens": 423, + "val/ratio": 0.9886413812637329, + "val/ratio_var": 8.117071411106735e-05 + }, + { + "episode": 17040, + "epoch": 0.8606060606060606, + "eps": 0, + "loss/policy_avg": -0.024640943855047226, + "loss/value_avg": 0.736940324306488, + "lr": 5.305054389629022e-07, + "objective/entropy": -66.94715881347656, + "objective/kl": 41.044742584228516, + "objective/non_score_reward": -2.052237033843994, + "objective/rlhf_reward": 13.018075942993164, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009623835212551057, + "policy/clipfrac_avg": 0.0037963923532515764, + "policy/entropy_avg": 0.1932220458984375, + "step": 1065, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 86, + "val/ratio": 0.9877476692199707, + "val/ratio_var": 5.7415742048760876e-05 + }, + { + "episode": 17056, + "epoch": 0.8614141414141414, + "eps": 0, + "loss/policy_avg": 0.11599922180175781, + "loss/value_avg": 1.5807065963745117, + "lr": 5.245338206604733e-07, + "objective/entropy": -115.46347045898438, + "objective/kl": 40.31273651123047, + "objective/non_score_reward": -2.015636682510376, + "objective/rlhf_reward": 13.367176055908203, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008412263705395162, + "policy/clipfrac_avg": 0.01834527961909771, + "policy/entropy_avg": 0.17316818237304688, + "step": 1066, + "val/clipfrac_avg": 1.0643732821336016e-05, + "val/num_eos_tokens": 71, + "val/ratio": 0.9887286424636841, + "val/ratio_var": 4.431970592122525e-05 + }, + { + "episode": 17072, + "epoch": 0.8622222222222222, + "eps": 0, + "loss/policy_avg": -0.11949857324361801, + "loss/value_avg": 0.5354464650154114, + "lr": 5.185941414910673e-07, + "objective/entropy": -67.14189147949219, + "objective/kl": 39.36028289794922, + "objective/non_score_reward": -1.9680140018463135, + "objective/rlhf_reward": 15.414798736572266, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0008963524596765637, + "policy/clipfrac_avg": 0.00728583661839366, + "policy/entropy_avg": 0.17730861902236938, + "step": 1067, + "val/clipfrac_avg": 0.0072115384973585606, + "val/num_eos_tokens": 286, + "val/ratio": 0.9881836175918579, + "val/ratio_var": 3.5629829653771594e-05 + }, + { + "episode": 17088, + "epoch": 0.863030303030303, + "eps": 0, + "loss/policy_avg": 0.10199490189552307, + "loss/value_avg": 0.35868531465530396, + "lr": 5.126864438431628e-07, + "objective/entropy": -115.96305847167969, + "objective/kl": 37.194644927978516, + "objective/non_score_reward": -1.8597323894500732, + "objective/rlhf_reward": 12.882455825805664, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000633776537142694, + "policy/clipfrac_avg": 0.004217084962874651, + "policy/entropy_avg": 0.13162803649902344, + "step": 1068, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 156, + "val/ratio": 0.9914271235466003, + "val/ratio_var": 5.447517105494626e-05 + }, + { + "episode": 17104, + "epoch": 0.8638383838383838, + "eps": 0, + "loss/policy_avg": 0.09657607972621918, + "loss/value_avg": 0.33049535751342773, + "lr": 5.06810769877002e-07, + "objective/entropy": -27.455425262451172, + "objective/kl": 30.918790817260742, + "objective/non_score_reward": -1.545939564704895, + "objective/rlhf_reward": 13.969684600830078, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000932033930439502, + "policy/clipfrac_avg": 0.0023164437152445316, + "policy/entropy_avg": 0.2050342559814453, + "step": 1069, + "val/clipfrac_avg": 0.0017710273386910558, + "val/num_eos_tokens": 117, + "val/ratio": 0.9869778752326965, + "val/ratio_var": 7.318282587220892e-05 + }, + { + "episode": 17120, + "epoch": 0.8646464646464647, + "eps": 0, + "loss/policy_avg": -0.07673387229442596, + "loss/value_avg": 0.6429687738418579, + "lr": 5.009671615242928e-07, + "objective/entropy": -41.616695404052734, + "objective/kl": 42.76515197753906, + "objective/non_score_reward": -2.1382575035095215, + "objective/rlhf_reward": 13.17424201965332, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009901263983920217, + "policy/clipfrac_avg": 0.003959751222282648, + "policy/entropy_avg": 0.1980295181274414, + "step": 1070, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 199, + "val/ratio": 0.987491250038147, + "val/ratio_var": 4.666786844609305e-05 + }, + { + "episode": 17136, + "epoch": 0.8654545454545455, + "eps": 0, + "loss/policy_avg": -0.03734193369746208, + "loss/value_avg": 0.7440835237503052, + "lr": 4.951556604879049e-07, + "objective/entropy": -53.619712829589844, + "objective/kl": 39.59283447265625, + "objective/non_score_reward": -1.9796417951583862, + "objective/rlhf_reward": 13.887545585632324, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010703656589612365, + "policy/clipfrac_avg": 0.0033148752991110086, + "policy/entropy_avg": 0.19559097290039062, + "step": 1071, + "val/clipfrac_avg": 0.00032552084303461015, + "val/num_eos_tokens": 134, + "val/ratio": 0.9871270060539246, + "val/ratio_var": 6.861035944893956e-05 + }, + { + "episode": 17152, + "epoch": 0.8662626262626263, + "eps": 0, + "loss/policy_avg": 0.05410761386156082, + "loss/value_avg": 0.6214333772659302, + "lr": 4.893763082415736e-07, + "objective/entropy": -32.19466781616211, + "objective/kl": 33.41657638549805, + "objective/non_score_reward": -1.6708288192749023, + "objective/rlhf_reward": 12.516671180725098, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010530936997383833, + "policy/clipfrac_avg": 0.005051491782069206, + "policy/entropy_avg": 0.18621063232421875, + "step": 1072, + "val/clipfrac_avg": 0.006988893263041973, + "val/num_eos_tokens": 41, + "val/ratio": 0.9861816167831421, + "val/ratio_var": 5.813246752950363e-05 + }, + { + "episode": 17168, + "epoch": 0.8670707070707071, + "eps": 0, + "loss/policy_avg": -0.13285210728645325, + "loss/value_avg": 0.37682080268859863, + "lr": 4.836291460296066e-07, + "objective/entropy": -73.75119018554688, + "objective/kl": 44.275638580322266, + "objective/non_score_reward": -2.2137818336486816, + "objective/rlhf_reward": 13.872156143188477, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009736482752487063, + "policy/clipfrac_avg": 0.002118697389960289, + "policy/entropy_avg": 0.1989288330078125, + "step": 1073, + "val/clipfrac_avg": 0.003361742477864027, + "val/num_eos_tokens": 178, + "val/ratio": 0.9864859580993652, + "val/ratio_var": 7.720785652054474e-05 + }, + { + "episode": 17184, + "epoch": 0.8678787878787879, + "eps": 0, + "loss/policy_avg": 0.032950110733509064, + "loss/value_avg": 0.5699096918106079, + "lr": 4.779142148665855e-07, + "objective/entropy": -17.03137969970703, + "objective/kl": 29.793800354003906, + "objective/non_score_reward": -1.4896900653839111, + "objective/rlhf_reward": 15.447810173034668, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.000868711736984551, + "policy/clipfrac_avg": 0.0015887501649558544, + "policy/entropy_avg": 0.17066001892089844, + "step": 1074, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 84, + "val/ratio": 0.9895613193511963, + "val/ratio_var": 7.112583261914551e-05 + }, + { + "episode": 17200, + "epoch": 0.8686868686868687, + "eps": 0, + "loss/policy_avg": -0.11098457127809525, + "loss/value_avg": 0.5394589304924011, + "lr": 4.722315555370793e-07, + "objective/entropy": -46.34983825683594, + "objective/kl": 39.953025817871094, + "objective/non_score_reward": -1.997651219367981, + "objective/rlhf_reward": 14.674223899841309, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008599478751420975, + "policy/clipfrac_avg": 0.006935285869985819, + "policy/entropy_avg": 0.17142105102539062, + "step": 1075, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 73, + "val/ratio": 0.9890701770782471, + "val/ratio_var": 3.540362376952544e-05 + }, + { + "episode": 17216, + "epoch": 0.8694949494949495, + "eps": 0, + "loss/policy_avg": 0.11330074071884155, + "loss/value_avg": 0.47666221857070923, + "lr": 4.665812085953425e-07, + "objective/entropy": -62.084415435791016, + "objective/kl": 43.93402099609375, + "objective/non_score_reward": -2.1967010498046875, + "objective/rlhf_reward": 12.162673950195312, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009052088716998696, + "policy/clipfrac_avg": 0.0032377864699810743, + "policy/entropy_avg": 0.18408584594726562, + "step": 1076, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 588, + "val/ratio": 0.9876794815063477, + "val/ratio_var": 4.1342565964441746e-05 + }, + { + "episode": 17232, + "epoch": 0.8703030303030304, + "eps": 0, + "loss/policy_avg": -0.05885958671569824, + "loss/value_avg": 0.9560117125511169, + "lr": 4.6096321436504e-07, + "objective/entropy": -73.24485778808594, + "objective/kl": 39.06366729736328, + "objective/non_score_reward": -1.9531835317611694, + "objective/rlhf_reward": 13.5468168258667, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009686506818979979, + "policy/clipfrac_avg": 0.004011271521449089, + "policy/entropy_avg": 0.18913555145263672, + "step": 1077, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 200, + "val/ratio": 0.9870175719261169, + "val/ratio_var": 8.096115197986364e-05 + }, + { + "episode": 17248, + "epoch": 0.8711111111111111, + "eps": 0, + "loss/policy_avg": 0.18471674621105194, + "loss/value_avg": 0.37227392196655273, + "lr": 4.5537761293894535e-07, + "objective/entropy": -116.47470092773438, + "objective/kl": 37.137413024902344, + "objective/non_score_reward": -1.8568708896636963, + "objective/rlhf_reward": 14.861879348754883, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000727109843865037, + "policy/clipfrac_avg": 0.002056294586509466, + "policy/entropy_avg": 0.1612076759338379, + "step": 1078, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 178, + "val/ratio": 0.9902770519256592, + "val/ratio_var": 6.446443876484409e-05 + }, + { + "episode": 17264, + "epoch": 0.8719191919191919, + "eps": 0, + "loss/policy_avg": -0.16059936583042145, + "loss/value_avg": 0.507465124130249, + "lr": 4.4982444417866753e-07, + "objective/entropy": -93.82952880859375, + "objective/kl": 37.85641098022461, + "objective/non_score_reward": -1.8928205966949463, + "objective/rlhf_reward": 13.458742141723633, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0009164538932964206, + "policy/clipfrac_avg": 0.0050585465505719185, + "policy/entropy_avg": 0.15972977876663208, + "step": 1079, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 44, + "val/ratio": 0.989949643611908, + "val/ratio_var": 8.05418094387278e-05 + }, + { + "episode": 17280, + "epoch": 0.8727272727272727, + "eps": 0, + "loss/policy_avg": 0.024168267846107483, + "loss/value_avg": 0.5276412963867188, + "lr": 4.4430374771435245e-07, + "objective/entropy": -93.29814910888672, + "objective/kl": 36.883766174316406, + "objective/non_score_reward": -1.8441884517669678, + "objective/rlhf_reward": 13.952686309814453, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0010746137704700232, + "policy/clipfrac_avg": 0.005830212030559778, + "policy/entropy_avg": 0.1792011260986328, + "step": 1080, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 519, + "val/ratio": 0.989915132522583, + "val/ratio_var": 4.745582918985747e-05 + }, + { + "episode": 17296, + "epoch": 0.8735353535353535, + "eps": 0, + "loss/policy_avg": -0.09750235080718994, + "loss/value_avg": 0.4400123357772827, + "lr": 4.3881556294441516e-07, + "objective/entropy": -163.2279815673828, + "objective/kl": 42.45173645019531, + "objective/non_score_reward": -2.122586727142334, + "objective/rlhf_reward": 13.236787796020508, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007963902316987514, + "policy/clipfrac_avg": 0.0037931441329419613, + "policy/entropy_avg": 0.17099809646606445, + "step": 1081, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 294, + "val/ratio": 0.9886674880981445, + "val/ratio_var": 8.438028453383595e-05 + }, + { + "episode": 17312, + "epoch": 0.8743434343434343, + "eps": 0, + "loss/policy_avg": -0.0659959465265274, + "loss/value_avg": 0.2975969910621643, + "lr": 4.333599290352458e-07, + "objective/entropy": -63.06423568725586, + "objective/kl": 42.34510803222656, + "objective/non_score_reward": -2.117255210876465, + "objective/rlhf_reward": 14.288995742797852, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009070255910046399, + "policy/clipfrac_avg": 0.0022676163353025913, + "policy/entropy_avg": 0.18258285522460938, + "step": 1082, + "val/clipfrac_avg": 0.0035635964013636112, + "val/num_eos_tokens": 576, + "val/ratio": 0.9886539578437805, + "val/ratio_var": 6.365034641930833e-05 + }, + { + "episode": 17328, + "epoch": 0.8751515151515151, + "eps": 0, + "loss/policy_avg": -0.12663836777210236, + "loss/value_avg": 1.6143395900726318, + "lr": 4.279368849209381e-07, + "objective/entropy": -124.63575744628906, + "objective/kl": 34.96723937988281, + "objective/non_score_reward": -1.7483620643615723, + "objective/rlhf_reward": 15.900075912475586, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0008090031915344298, + "policy/clipfrac_avg": 0.0031590727157890797, + "policy/entropy_avg": 0.16525745391845703, + "step": 1083, + "val/clipfrac_avg": 0.02003205195069313, + "val/num_eos_tokens": 359, + "val/ratio": 0.9890646934509277, + "val/ratio_var": 7.246930908877403e-05 + }, + { + "episode": 17344, + "epoch": 0.8759595959595959, + "eps": 0, + "loss/policy_avg": 0.10138820856809616, + "loss/value_avg": 0.3533989191055298, + "lr": 4.2254646930300845e-07, + "objective/entropy": -55.43729019165039, + "objective/kl": 44.472023010253906, + "objective/non_score_reward": -2.2236011028289795, + "objective/rlhf_reward": 13.721712112426758, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009467223426327109, + "policy/clipfrac_avg": 0.0063713230192661285, + "policy/entropy_avg": 0.1949615478515625, + "step": 1084, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.986469030380249, + "val/ratio_var": 5.372995656216517e-05 + }, + { + "episode": 17360, + "epoch": 0.8767676767676768, + "eps": 0, + "loss/policy_avg": -0.16358020901679993, + "loss/value_avg": 0.7672687768936157, + "lr": 4.171887206501191e-07, + "objective/entropy": -80.793701171875, + "objective/kl": 35.82070541381836, + "objective/non_score_reward": -1.7910352945327759, + "objective/rlhf_reward": 13.552715301513672, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007973198080435395, + "policy/clipfrac_avg": 0.003301304765045643, + "policy/entropy_avg": 0.15000247955322266, + "step": 1085, + "val/clipfrac_avg": 0.0021186440717428923, + "val/num_eos_tokens": 403, + "val/ratio": 0.9896234273910522, + "val/ratio_var": 5.3207288146950305e-05 + }, + { + "episode": 17376, + "epoch": 0.8775757575757576, + "eps": 0, + "loss/policy_avg": 0.20687676966190338, + "loss/value_avg": 1.3663004636764526, + "lr": 4.1186367719780737e-07, + "objective/entropy": -46.82775115966797, + "objective/kl": 33.1654167175293, + "objective/non_score_reward": -1.6582709550857544, + "objective/rlhf_reward": 14.404230117797852, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008230544626712799, + "policy/clipfrac_avg": 0.0030164497438818216, + "policy/entropy_avg": 0.1555004119873047, + "step": 1086, + "val/clipfrac_avg": 0.0026466259732842445, + "val/num_eos_tokens": 47, + "val/ratio": 0.9892673492431641, + "val/ratio_var": 3.7006568163633347e-05 + }, + { + "episode": 17392, + "epoch": 0.8783838383838384, + "eps": 0, + "loss/policy_avg": 0.18984848260879517, + "loss/value_avg": 0.31801366806030273, + "lr": 4.0657137694820826e-07, + "objective/entropy": -66.93091583251953, + "objective/kl": 45.35851287841797, + "objective/non_score_reward": -2.26792573928833, + "objective/rlhf_reward": 14.122699737548828, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008988711051642895, + "policy/clipfrac_avg": 0.004365711007267237, + "policy/entropy_avg": 0.17530107498168945, + "step": 1087, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 363, + "val/ratio": 0.9879393577575684, + "val/ratio_var": 6.273078179219738e-05 + }, + { + "episode": 17408, + "epoch": 0.8791919191919192, + "eps": 0, + "loss/policy_avg": 0.14154401421546936, + "loss/value_avg": 0.296213299036026, + "lr": 4.013118576697844e-07, + "objective/entropy": -96.627197265625, + "objective/kl": 44.89594650268555, + "objective/non_score_reward": -2.244797468185425, + "objective/rlhf_reward": 14.028639793395996, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009503285982646048, + "policy/clipfrac_avg": 0.004713431932032108, + "policy/entropy_avg": 0.19094181060791016, + "step": 1088, + "val/clipfrac_avg": 0.004036702681332827, + "val/num_eos_tokens": 215, + "val/ratio": 0.9875110983848572, + "val/ratio_var": 3.645908509497531e-05 + }, + { + "episode": 17424, + "epoch": 0.88, + "eps": 0, + "loss/policy_avg": -0.2717769145965576, + "loss/value_avg": 0.6515541076660156, + "lr": 3.960851568970586e-07, + "objective/entropy": -81.71890258789062, + "objective/kl": 44.09088897705078, + "objective/non_score_reward": -2.2045445442199707, + "objective/rlhf_reward": 14.311080932617188, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008286006050184369, + "policy/clipfrac_avg": 0.0029842001385986805, + "policy/entropy_avg": 0.16189241409301758, + "step": 1089, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 391, + "val/ratio": 0.9892988204956055, + "val/ratio_var": 5.128037810209207e-05 + }, + { + "episode": 17440, + "epoch": 0.8808080808080808, + "eps": 0, + "loss/policy_avg": -0.09007427841424942, + "loss/value_avg": 0.174169659614563, + "lr": 3.908913119303448e-07, + "objective/entropy": -103.39527893066406, + "objective/kl": 45.33392333984375, + "objective/non_score_reward": -2.2666964530944824, + "objective/rlhf_reward": 15.030179023742676, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008385264663957059, + "policy/clipfrac_avg": 0.0022599862422794104, + "policy/entropy_avg": 0.167741060256958, + "step": 1090, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9883161783218384, + "val/ratio_var": 4.988587534171529e-05 + }, + { + "episode": 17456, + "epoch": 0.8816161616161616, + "eps": 0, + "loss/policy_avg": 0.29356396198272705, + "loss/value_avg": 0.8743624687194824, + "lr": 3.8573035983548167e-07, + "objective/entropy": -82.11065673828125, + "objective/kl": 38.23064422607422, + "objective/non_score_reward": -1.9115324020385742, + "objective/rlhf_reward": 13.299405097961426, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008716207230463624, + "policy/clipfrac_avg": 0.0072744423523545265, + "policy/entropy_avg": 0.16114044189453125, + "step": 1091, + "val/clipfrac_avg": 0.0008928571478463709, + "val/num_eos_tokens": 91, + "val/ratio": 0.9882928133010864, + "val/ratio_var": 6.514011329272762e-05 + }, + { + "episode": 17472, + "epoch": 0.8824242424242424, + "eps": 0, + "loss/policy_avg": 0.02994067780673504, + "loss/value_avg": 0.792471170425415, + "lr": 3.8060233744356634e-07, + "objective/entropy": -12.753183364868164, + "objective/kl": 37.99050521850586, + "objective/non_score_reward": -1.8995252847671509, + "objective/rlhf_reward": 12.65125560760498, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0010071105789393187, + "policy/clipfrac_avg": 0.004005424678325653, + "policy/entropy_avg": 0.19425201416015625, + "step": 1092, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 14, + "val/ratio": 0.9855411052703857, + "val/ratio_var": 4.6690693125128746e-05 + }, + { + "episode": 17488, + "epoch": 0.8832323232323233, + "eps": 0, + "loss/policy_avg": 0.1304478645324707, + "loss/value_avg": 0.3748246431350708, + "lr": 3.7550728135069545e-07, + "objective/entropy": -36.16312789916992, + "objective/kl": 48.05560302734375, + "objective/non_score_reward": -2.402780532836914, + "objective/rlhf_reward": 14.300344467163086, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000979696400463581, + "policy/clipfrac_avg": 0.005794232711195946, + "policy/entropy_avg": 0.19660568237304688, + "step": 1093, + "val/clipfrac_avg": 0.0026041667442768812, + "val/num_eos_tokens": 66, + "val/ratio": 0.9865772128105164, + "val/ratio_var": 5.429132215795107e-05 + }, + { + "episode": 17504, + "epoch": 0.8840404040404041, + "eps": 0, + "loss/policy_avg": 0.07484120875597, + "loss/value_avg": 0.2602160573005676, + "lr": 3.704452279177012e-07, + "objective/entropy": -71.43814086914062, + "objective/kl": 45.28865051269531, + "objective/non_score_reward": -2.264432668685913, + "objective/rlhf_reward": 13.727754592895508, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000950466375797987, + "policy/clipfrac_avg": 0.0027152306865900755, + "policy/entropy_avg": 0.20662879943847656, + "step": 1094, + "val/clipfrac_avg": 0.0006720430101267993, + "val/num_eos_tokens": 494, + "val/ratio": 0.9866384863853455, + "val/ratio_var": 7.357261347351596e-05 + }, + { + "episode": 17520, + "epoch": 0.8848484848484849, + "eps": 0, + "loss/policy_avg": 0.10963790118694305, + "loss/value_avg": 0.7320235967636108, + "lr": 3.6541621326989183e-07, + "objective/entropy": -48.0823974609375, + "objective/kl": 39.92009353637695, + "objective/non_score_reward": -1.9960048198699951, + "objective/rlhf_reward": 12.980557441711426, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0010038036853075027, + "policy/clipfrac_avg": 0.004399286117404699, + "policy/entropy_avg": 0.19763946533203125, + "step": 1095, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 436, + "val/ratio": 0.987025260925293, + "val/ratio_var": 3.8341207982739434e-05 + }, + { + "episode": 17536, + "epoch": 0.8856565656565657, + "eps": 0, + "loss/policy_avg": 0.05299535021185875, + "loss/value_avg": 0.2717387080192566, + "lr": 3.6042027329679487e-07, + "objective/entropy": -51.004615783691406, + "objective/kl": 41.74889373779297, + "objective/non_score_reward": -2.087444543838501, + "objective/rlhf_reward": 14.545368194580078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010824615601450205, + "policy/clipfrac_avg": 0.00963993463665247, + "policy/entropy_avg": 0.20147705078125, + "step": 1096, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 345, + "val/ratio": 0.9875917434692383, + "val/ratio_var": 6.199166091391817e-05 + }, + { + "episode": 17552, + "epoch": 0.8864646464646465, + "eps": 0, + "loss/policy_avg": -0.08447069674730301, + "loss/value_avg": 0.38099169731140137, + "lr": 3.5545744365190204e-07, + "objective/entropy": -108.7376937866211, + "objective/kl": 41.82084655761719, + "objective/non_score_reward": -2.0910425186157227, + "objective/rlhf_reward": 14.158958435058594, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000758812646381557, + "policy/clipfrac_avg": 0.003666321514174342, + "policy/entropy_avg": 0.15442466735839844, + "step": 1097, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 244, + "val/ratio": 0.9902445077896118, + "val/ratio_var": 2.8433618354029022e-05 + }, + { + "episode": 17568, + "epoch": 0.8872727272727273, + "eps": 0, + "loss/policy_avg": -0.47139230370521545, + "loss/value_avg": 1.4234567880630493, + "lr": 3.5052775975241203e-07, + "objective/entropy": -131.56527709960938, + "objective/kl": 33.65018081665039, + "objective/non_score_reward": -1.682509183883667, + "objective/rlhf_reward": 13.74717903137207, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009085559286177158, + "policy/clipfrac_avg": 0.004985325504094362, + "policy/entropy_avg": 0.16993939876556396, + "step": 1098, + "val/clipfrac_avg": 0.00024242099607363343, + "val/num_eos_tokens": 1220, + "val/ratio": 0.9888011813163757, + "val/ratio_var": 7.847072265576571e-05 + }, + { + "episode": 17584, + "epoch": 0.888080808080808, + "eps": 0, + "loss/policy_avg": 0.16145119071006775, + "loss/value_avg": 0.5360366106033325, + "lr": 3.4563125677897936e-07, + "objective/entropy": -42.34114074707031, + "objective/kl": 37.358299255371094, + "objective/non_score_reward": -1.867915153503418, + "objective/rlhf_reward": 14.428959846496582, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001106644980609417, + "policy/clipfrac_avg": 0.0073922015726566315, + "policy/entropy_avg": 0.216278076171875, + "step": 1099, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 172, + "val/ratio": 0.9858206510543823, + "val/ratio_var": 6.1017286498099566e-05 + }, + { + "episode": 17600, + "epoch": 0.8888888888888888, + "eps": 0, + "loss/policy_avg": 0.043675780296325684, + "loss/value_avg": 0.5687451362609863, + "lr": 3.4076796967546176e-07, + "objective/entropy": -32.0341682434082, + "objective/kl": 43.24127197265625, + "objective/non_score_reward": -2.1620638370513916, + "objective/rlhf_reward": 12.939498901367188, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010629122843965888, + "policy/clipfrac_avg": 0.003724365960806608, + "policy/entropy_avg": 0.20337867736816406, + "step": 1100, + "val/clipfrac_avg": 0.005150945391505957, + "val/num_eos_tokens": 106, + "val/ratio": 0.9857684969902039, + "val/ratio_var": 4.0027771319728345e-05 + }, + { + "episode": 17616, + "epoch": 0.8896969696969697, + "eps": 0, + "loss/policy_avg": -0.0018694326281547546, + "loss/value_avg": 0.43309682607650757, + "lr": 3.359379331486762e-07, + "objective/entropy": -65.23380279541016, + "objective/kl": 40.037052154541016, + "objective/non_score_reward": -2.001852512359619, + "objective/rlhf_reward": 13.349709510803223, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010041861096397042, + "policy/clipfrac_avg": 0.004063514992594719, + "policy/entropy_avg": 0.19306087493896484, + "step": 1101, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 14, + "val/ratio": 0.9867812395095825, + "val/ratio_var": 5.167184644960798e-05 + }, + { + "episode": 17632, + "epoch": 0.8905050505050505, + "eps": 0, + "loss/policy_avg": 0.048976313322782516, + "loss/value_avg": 0.3672630190849304, + "lr": 3.311411816681426e-07, + "objective/entropy": -81.65939331054688, + "objective/kl": 41.81010055541992, + "objective/non_score_reward": -2.0905051231384277, + "objective/rlhf_reward": 14.393869400024414, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000925385276786983, + "policy/clipfrac_avg": 0.004391741007566452, + "policy/entropy_avg": 0.1823406219482422, + "step": 1102, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 128, + "val/ratio": 0.9886959791183472, + "val/ratio_var": 5.684143616235815e-05 + }, + { + "episode": 17648, + "epoch": 0.8913131313131313, + "eps": 0, + "loss/policy_avg": 0.02880220115184784, + "loss/value_avg": 0.3456343114376068, + "lr": 3.263777494658449e-07, + "objective/entropy": -97.10613250732422, + "objective/kl": 39.38318634033203, + "objective/non_score_reward": -1.9691593647003174, + "objective/rlhf_reward": 14.226152420043945, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008451448520645499, + "policy/clipfrac_avg": 0.0035835830494761467, + "policy/entropy_avg": 0.18262100219726562, + "step": 1103, + "val/clipfrac_avg": 0.0004976114723831415, + "val/num_eos_tokens": 81, + "val/ratio": 0.9880388975143433, + "val/ratio_var": 4.812321640201844e-05 + }, + { + "episode": 17664, + "epoch": 0.8921212121212121, + "eps": 0, + "loss/policy_avg": -0.15117420256137848, + "loss/value_avg": 1.1375534534454346, + "lr": 3.216476705359839e-07, + "objective/entropy": -138.00392150878906, + "objective/kl": 39.81349182128906, + "objective/non_score_reward": -1.990674614906311, + "objective/rlhf_reward": 13.14213752746582, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009344244026578963, + "policy/clipfrac_avg": 0.004199904855340719, + "policy/entropy_avg": 0.1756887435913086, + "step": 1104, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 173, + "val/ratio": 0.9869411587715149, + "val/ratio_var": 8.181409793905914e-05 + }, + { + "episode": 17680, + "epoch": 0.8929292929292929, + "eps": 0, + "loss/policy_avg": -0.40621763467788696, + "loss/value_avg": 0.46248072385787964, + "lr": 3.1695097863473355e-07, + "objective/entropy": -191.18865966796875, + "objective/kl": 38.82806396484375, + "objective/non_score_reward": -1.9414031505584717, + "objective/rlhf_reward": 13.941408157348633, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0005901164840906858, + "policy/clipfrac_avg": 0.005352393724024296, + "policy/entropy_avg": 0.1206204891204834, + "step": 1105, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 236, + "val/ratio": 0.9924303293228149, + "val/ratio_var": 4.216426896164194e-05 + }, + { + "episode": 17696, + "epoch": 0.8937373737373737, + "eps": 0, + "loss/policy_avg": -0.11594279855489731, + "loss/value_avg": 0.7615197896957397, + "lr": 3.122877072800046e-07, + "objective/entropy": -87.4638900756836, + "objective/kl": 37.186309814453125, + "objective/non_score_reward": -1.8593155145645142, + "objective/rlhf_reward": 15.757871627807617, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0008562589064240456, + "policy/clipfrac_avg": 0.0030250055715441704, + "policy/entropy_avg": 0.1770009994506836, + "step": 1106, + "val/clipfrac_avg": 8.561644062865525e-05, + "val/num_eos_tokens": 557, + "val/ratio": 0.9886528253555298, + "val/ratio_var": 7.094601460266858e-05 + }, + { + "episode": 17712, + "epoch": 0.8945454545454545, + "eps": 0, + "loss/policy_avg": -0.12702879309654236, + "loss/value_avg": 0.3219323754310608, + "lr": 3.076578897511978e-07, + "objective/entropy": -110.63394927978516, + "objective/kl": 39.78605651855469, + "objective/non_score_reward": -1.9893031120300293, + "objective/rlhf_reward": 14.362258911132812, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007805367931723595, + "policy/clipfrac_avg": 0.0034728883765637875, + "policy/entropy_avg": 0.14988577365875244, + "step": 1107, + "val/clipfrac_avg": 0.00021701389050576836, + "val/num_eos_tokens": 344, + "val/ratio": 0.9894638657569885, + "val/ratio_var": 6.243310053832829e-05 + }, + { + "episode": 17728, + "epoch": 0.8953535353535353, + "eps": 0, + "loss/policy_avg": -0.030154824256896973, + "loss/value_avg": 0.6115067005157471, + "lr": 3.0306155908897626e-07, + "objective/entropy": -74.73516845703125, + "objective/kl": 35.881317138671875, + "objective/non_score_reward": -1.7940657138824463, + "objective/rlhf_reward": 14.002809524536133, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009087615762837231, + "policy/clipfrac_avg": 0.004955953918397427, + "policy/entropy_avg": 0.1930408477783203, + "step": 1108, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 300, + "val/ratio": 0.9869700074195862, + "val/ratio_var": 6.0412487073335797e-05 + }, + { + "episode": 17744, + "epoch": 0.8961616161616162, + "eps": 0, + "loss/policy_avg": -0.01582721620798111, + "loss/value_avg": 0.5954544544219971, + "lr": 2.984987480950191e-07, + "objective/entropy": -105.74530029296875, + "objective/kl": 33.56185531616211, + "objective/non_score_reward": -1.6780929565429688, + "objective/rlhf_reward": 12.939094543457031, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009151956764981151, + "policy/clipfrac_avg": 0.004283002577722073, + "policy/entropy_avg": 0.18004417419433594, + "step": 1109, + "val/clipfrac_avg": 0.001953125, + "val/num_eos_tokens": 107, + "val/ratio": 0.9886264801025391, + "val/ratio_var": 6.606351962545887e-05 + }, + { + "episode": 17760, + "epoch": 0.896969696969697, + "eps": 0, + "loss/policy_avg": -0.0613558366894722, + "loss/value_avg": 0.37346547842025757, + "lr": 2.939694893317979e-07, + "objective/entropy": -105.83699798583984, + "objective/kl": 40.10255432128906, + "objective/non_score_reward": -2.0051279067993164, + "objective/rlhf_reward": 13.768309593200684, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009933009278029203, + "policy/clipfrac_avg": 0.002607613569125533, + "policy/entropy_avg": 0.19908523559570312, + "step": 1110, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 189, + "val/ratio": 0.9867061376571655, + "val/ratio_var": 8.13972728792578e-05 + }, + { + "episode": 17776, + "epoch": 0.8977777777777778, + "eps": 0, + "loss/policy_avg": 0.0873388946056366, + "loss/value_avg": 0.6430274844169617, + "lr": 2.894738151223331e-07, + "objective/entropy": -69.66899108886719, + "objective/kl": 45.03960418701172, + "objective/non_score_reward": -2.2519803047180176, + "objective/rlhf_reward": 15.544894218444824, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0010472675785422325, + "policy/clipfrac_avg": 0.0032264026813209057, + "policy/entropy_avg": 0.20671892166137695, + "step": 1111, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 714, + "val/ratio": 0.9859870672225952, + "val/ratio_var": 6.778486567782238e-05 + }, + { + "episode": 17792, + "epoch": 0.8985858585858586, + "eps": 0, + "loss/policy_avg": -0.13560356199741364, + "loss/value_avg": 0.6080023646354675, + "lr": 2.850117575499739e-07, + "objective/entropy": -89.71099090576172, + "objective/kl": 39.79743957519531, + "objective/non_score_reward": -1.9898720979690552, + "objective/rlhf_reward": 16.385128021240234, + "objective/scores": 18.375, + "policy/approxkl_avg": 0.00102079042699188, + "policy/clipfrac_avg": 0.004375361371785402, + "policy/entropy_avg": 0.19225120544433594, + "step": 1112, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 296, + "val/ratio": 0.9872239828109741, + "val/ratio_var": 7.309510692721233e-05 + }, + { + "episode": 17808, + "epoch": 0.8993939393939394, + "eps": 0, + "loss/policy_avg": -0.33778393268585205, + "loss/value_avg": 0.4939768612384796, + "lr": 2.8058334845816214e-07, + "objective/entropy": -31.377525329589844, + "objective/kl": 48.40119934082031, + "objective/non_score_reward": -2.4200599193573, + "objective/rlhf_reward": 15.626814842224121, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010773141402751207, + "policy/clipfrac_avg": 0.007447387091815472, + "policy/entropy_avg": 0.2159743309020996, + "step": 1113, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 28, + "val/ratio": 0.9851824641227722, + "val/ratio_var": 7.278792327269912e-05 + }, + { + "episode": 17824, + "epoch": 0.9002020202020202, + "eps": 0, + "loss/policy_avg": -0.1028350442647934, + "loss/value_avg": 0.5992170572280884, + "lr": 2.761886194502089e-07, + "objective/entropy": -30.16031265258789, + "objective/kl": 37.62234115600586, + "objective/non_score_reward": -1.8811171054840088, + "objective/rlhf_reward": 12.79857063293457, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010548201389610767, + "policy/clipfrac_avg": 0.0043435124680399895, + "policy/entropy_avg": 0.19416046142578125, + "step": 1114, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 115, + "val/ratio": 0.9875701665878296, + "val/ratio_var": 4.415150760905817e-05 + }, + { + "episode": 17840, + "epoch": 0.901010101010101, + "eps": 0, + "loss/policy_avg": 0.15831202268600464, + "loss/value_avg": 0.5015832185745239, + "lr": 2.71827601889067e-07, + "objective/entropy": -98.54946899414062, + "objective/kl": 44.19203567504883, + "objective/non_score_reward": -2.209601879119873, + "objective/rlhf_reward": 12.516960144042969, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0008550393395125866, + "policy/clipfrac_avg": 0.0035258647985756397, + "policy/entropy_avg": 0.17531871795654297, + "step": 1115, + "val/clipfrac_avg": 9.245562250725925e-05, + "val/num_eos_tokens": 277, + "val/ratio": 0.9883965253829956, + "val/ratio_var": 6.70754598104395e-05 + }, + { + "episode": 17856, + "epoch": 0.9018181818181819, + "eps": 0, + "loss/policy_avg": -0.0883672684431076, + "loss/value_avg": 0.38530808687210083, + "lr": 2.6750032689710604e-07, + "objective/entropy": -89.13906860351562, + "objective/kl": 40.806068420410156, + "objective/non_score_reward": -2.0403032302856445, + "objective/rlhf_reward": 14.842509269714355, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009749368182383478, + "policy/clipfrac_avg": 0.004587238188832998, + "policy/entropy_avg": 0.1862497329711914, + "step": 1116, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.9878228902816772, + "val/ratio_var": 6.971527182031423e-05 + }, + { + "episode": 17872, + "epoch": 0.9026262626262627, + "eps": 0, + "loss/policy_avg": 0.35861504077911377, + "loss/value_avg": 0.47818782925605774, + "lr": 2.6320682535589557e-07, + "objective/entropy": -94.74272155761719, + "objective/kl": 39.73638916015625, + "objective/non_score_reward": -1.9868193864822388, + "objective/rlhf_reward": 14.48974323272705, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.001014375826343894, + "policy/clipfrac_avg": 0.009114467538893223, + "policy/entropy_avg": 0.18755674362182617, + "step": 1117, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 316, + "val/ratio": 0.9870187044143677, + "val/ratio_var": 6.580681656487286e-05 + }, + { + "episode": 17888, + "epoch": 0.9034343434343435, + "eps": 0, + "loss/policy_avg": 0.03886333107948303, + "loss/value_avg": 0.5049983263015747, + "lr": 2.589471279059791e-07, + "objective/entropy": -60.80832290649414, + "objective/kl": 38.03620147705078, + "objective/non_score_reward": -1.9018102884292603, + "objective/rlhf_reward": 14.418501853942871, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009664712124504149, + "policy/clipfrac_avg": 0.0054984488524496555, + "policy/entropy_avg": 0.18600749969482422, + "step": 1118, + "val/clipfrac_avg": 0.00028935185400769114, + "val/num_eos_tokens": 538, + "val/ratio": 0.9878861904144287, + "val/ratio_var": 8.505670120939612e-05 + }, + { + "episode": 17904, + "epoch": 0.9042424242424243, + "eps": 0, + "loss/policy_avg": -0.30915409326553345, + "loss/value_avg": 0.6310875415802002, + "lr": 2.547212649466568e-07, + "objective/entropy": -66.79815673828125, + "objective/kl": 40.48139190673828, + "objective/non_score_reward": -2.0240697860717773, + "objective/rlhf_reward": 13.483742713928223, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0010647093877196312, + "policy/clipfrac_avg": 0.005042470060288906, + "policy/entropy_avg": 0.2105545997619629, + "step": 1119, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 341, + "val/ratio": 0.9860644340515137, + "val/ratio_var": 7.680879207327962e-05 + }, + { + "episode": 17920, + "epoch": 0.9050505050505051, + "eps": 0, + "loss/policy_avg": -0.0354868620634079, + "loss/value_avg": 0.43115413188934326, + "lr": 2.5052926663577006e-07, + "objective/entropy": -109.19269561767578, + "objective/kl": 37.41370391845703, + "objective/non_score_reward": -1.8706854581832886, + "objective/rlhf_reward": 15.559001922607422, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0008096316596493125, + "policy/clipfrac_avg": 0.007008845917880535, + "policy/entropy_avg": 0.1526775360107422, + "step": 1120, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 27, + "val/ratio": 0.990079402923584, + "val/ratio_var": 6.870418292237446e-05 + }, + { + "episode": 17936, + "epoch": 0.9058585858585858, + "eps": 0, + "loss/policy_avg": -0.10384486615657806, + "loss/value_avg": 0.5555015802383423, + "lr": 2.4637116288948606e-07, + "objective/entropy": -81.66447448730469, + "objective/kl": 34.01490020751953, + "objective/non_score_reward": -1.7007449865341187, + "objective/rlhf_reward": 14.06488037109375, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011045493884012103, + "policy/clipfrac_avg": 0.004975738935172558, + "policy/entropy_avg": 0.19969654083251953, + "step": 1121, + "val/clipfrac_avg": 0.00020559210679493845, + "val/num_eos_tokens": 424, + "val/ratio": 0.9861068725585938, + "val/ratio_var": 0.0001140525855589658 + }, + { + "episode": 17952, + "epoch": 0.9066666666666666, + "eps": 0, + "loss/policy_avg": 0.08499134331941605, + "loss/value_avg": 0.4820760488510132, + "lr": 2.4224698338208344e-07, + "objective/entropy": -51.88709259033203, + "objective/kl": 36.03460693359375, + "objective/non_score_reward": -1.8017305135726929, + "objective/rlhf_reward": 13.284207344055176, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007865545339882374, + "policy/clipfrac_avg": 0.005190032534301281, + "policy/entropy_avg": 0.1644287109375, + "step": 1122, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 97, + "val/ratio": 0.9896594882011414, + "val/ratio_var": 3.428427953622304e-05 + }, + { + "episode": 17968, + "epoch": 0.9074747474747474, + "eps": 0, + "loss/policy_avg": -0.17406421899795532, + "loss/value_avg": 0.5896267294883728, + "lr": 2.3815675754573885e-07, + "objective/entropy": -86.18510437011719, + "objective/kl": 43.59098815917969, + "objective/non_score_reward": -2.179549217224121, + "objective/rlhf_reward": 14.422013282775879, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000971743487752974, + "policy/clipfrac_avg": 0.006423149257898331, + "policy/entropy_avg": 0.18558311462402344, + "step": 1123, + "val/clipfrac_avg": 0.0003633720916695893, + "val/num_eos_tokens": 28, + "val/ratio": 0.9873388409614563, + "val/ratio_var": 6.847102486062795e-05 + }, + { + "episode": 17984, + "epoch": 0.9082828282828282, + "eps": 0, + "loss/policy_avg": 0.003323908895254135, + "loss/value_avg": 0.4784785807132721, + "lr": 2.3410051457032179e-07, + "objective/entropy": -15.714594841003418, + "objective/kl": 46.467933654785156, + "objective/non_score_reward": -2.323396682739258, + "objective/rlhf_reward": 13.957853317260742, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009007846238091588, + "policy/clipfrac_avg": 0.004247956909239292, + "policy/entropy_avg": 0.18125152587890625, + "step": 1124, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 91, + "val/ratio": 0.9878742694854736, + "val/ratio_var": 4.734889080282301e-05 + }, + { + "episode": 18000, + "epoch": 0.9090909090909091, + "eps": 0, + "loss/policy_avg": -0.34681111574172974, + "loss/value_avg": 1.8761348724365234, + "lr": 2.3007828340318117e-07, + "objective/entropy": -78.62959289550781, + "objective/kl": 28.815372467041016, + "objective/non_score_reward": -1.4407685995101929, + "objective/rlhf_reward": 13.76235580444336, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0007939373608678579, + "policy/clipfrac_avg": 0.0023933430202305317, + "policy/entropy_avg": 0.14340460300445557, + "step": 1125, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 162, + "val/ratio": 0.9906671047210693, + "val/ratio_var": 8.598795102443546e-05 + }, + { + "episode": 18016, + "epoch": 0.9098989898989899, + "eps": 0, + "loss/policy_avg": -0.10182956606149673, + "loss/value_avg": 0.9262669682502747, + "lr": 2.2609009274894124e-07, + "objective/entropy": -84.69230651855469, + "objective/kl": 41.48188018798828, + "objective/non_score_reward": -2.07409405708313, + "objective/rlhf_reward": 14.402467727661133, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0007599142845720053, + "policy/clipfrac_avg": 0.004918557591736317, + "policy/entropy_avg": 0.14319229125976562, + "step": 1126, + "val/clipfrac_avg": 0.0003810975467786193, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904208779335022, + "val/ratio_var": 5.685169162461534e-05 + }, + { + "episode": 18032, + "epoch": 0.9107070707070707, + "eps": 0, + "loss/policy_avg": -0.05897387117147446, + "loss/value_avg": 0.6550663709640503, + "lr": 2.2213597106929608e-07, + "objective/entropy": 6.487635612487793, + "objective/kl": 36.32238006591797, + "objective/non_score_reward": -1.8161190748214722, + "objective/rlhf_reward": 12.402630805969238, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0012117292499169707, + "policy/clipfrac_avg": 0.003925529308617115, + "policy/entropy_avg": 0.22541046142578125, + "step": 1127, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 48, + "val/ratio": 0.9840712547302246, + "val/ratio_var": 4.843049100600183e-05 + }, + { + "episode": 18048, + "epoch": 0.9115151515151515, + "eps": 0, + "loss/policy_avg": -0.2118905931711197, + "loss/value_avg": 0.2701466381549835, + "lr": 2.1821594658280932e-07, + "objective/entropy": -81.51445007324219, + "objective/kl": 43.229087829589844, + "objective/non_score_reward": -2.161454439163208, + "objective/rlhf_reward": 14.604171752929688, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000821998284664005, + "policy/clipfrac_avg": 0.003186737420037389, + "policy/entropy_avg": 0.16611194610595703, + "step": 1128, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 379, + "val/ratio": 0.9885213971138, + "val/ratio_var": 5.360436989576556e-05 + }, + { + "episode": 18064, + "epoch": 0.9123232323232323, + "eps": 0, + "loss/policy_avg": 0.07222969084978104, + "loss/value_avg": 0.13441604375839233, + "lr": 2.1433004726470695e-07, + "objective/entropy": -90.69831085205078, + "objective/kl": 43.020301818847656, + "objective/non_score_reward": -2.151015281677246, + "objective/rlhf_reward": 15.84117317199707, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0008969155605882406, + "policy/clipfrac_avg": 0.004066394176334143, + "policy/entropy_avg": 0.18462800979614258, + "step": 1129, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 122, + "val/ratio": 0.9881325364112854, + "val/ratio_var": 5.7613899116404355e-05 + }, + { + "episode": 18080, + "epoch": 0.9131313131313131, + "eps": 0, + "loss/policy_avg": 0.030582871288061142, + "loss/value_avg": 0.32031866908073425, + "lr": 2.1047830084668264e-07, + "objective/entropy": -97.6414566040039, + "objective/kl": 39.517765045166016, + "objective/non_score_reward": -1.9758882522583008, + "objective/rlhf_reward": 13.8444242477417, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008600642322562635, + "policy/clipfrac_avg": 0.002656108234077692, + "policy/entropy_avg": 0.16113853454589844, + "step": 1130, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 730, + "val/ratio": 0.9883853793144226, + "val/ratio_var": 6.297288200585172e-05 + }, + { + "episode": 18096, + "epoch": 0.9139393939393939, + "eps": 0, + "loss/policy_avg": 0.004439752548933029, + "loss/value_avg": 0.5337626934051514, + "lr": 2.0666073481669714e-07, + "objective/entropy": -67.81413269042969, + "objective/kl": 34.70388412475586, + "objective/non_score_reward": -1.735194206237793, + "objective/rlhf_reward": 14.077305793762207, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009027072228491306, + "policy/clipfrac_avg": 0.004612314514815807, + "policy/entropy_avg": 0.18857192993164062, + "step": 1131, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 89, + "val/ratio": 0.9875516891479492, + "val/ratio_var": 7.778128201607615e-05 + }, + { + "episode": 18112, + "epoch": 0.9147474747474748, + "eps": 0, + "loss/policy_avg": -0.0649208128452301, + "loss/value_avg": 0.2573353052139282, + "lr": 2.0287737641878358e-07, + "objective/entropy": -57.457305908203125, + "objective/kl": 41.89903259277344, + "objective/non_score_reward": -2.094951629638672, + "objective/rlhf_reward": 15.764423370361328, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.001014683861285448, + "policy/clipfrac_avg": 0.005427089519798756, + "policy/entropy_avg": 0.2117767333984375, + "step": 1132, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 81, + "val/ratio": 0.985901415348053, + "val/ratio_var": 5.110340498504229e-05 + }, + { + "episode": 18128, + "epoch": 0.9155555555555556, + "eps": 0, + "loss/policy_avg": -0.03162732720375061, + "loss/value_avg": 0.41586339473724365, + "lr": 1.991282526528543e-07, + "objective/entropy": -32.120155334472656, + "objective/kl": 47.3048095703125, + "objective/non_score_reward": -2.3652405738830566, + "objective/rlhf_reward": 13.720697402954102, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010292984079569578, + "policy/clipfrac_avg": 0.004157859366387129, + "policy/entropy_avg": 0.20333099365234375, + "step": 1133, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 51, + "val/ratio": 0.9864168167114258, + "val/ratio_var": 5.427391442935914e-05 + }, + { + "episode": 18144, + "epoch": 0.9163636363636364, + "eps": 0, + "loss/policy_avg": 0.0864434763789177, + "loss/value_avg": 0.2746288478374481, + "lr": 1.9541339027450256e-07, + "objective/entropy": -68.75173950195312, + "objective/kl": 41.02958679199219, + "objective/non_score_reward": -2.0514793395996094, + "objective/rlhf_reward": 13.80789566040039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009240241488441825, + "policy/clipfrac_avg": 0.0033770925365388393, + "policy/entropy_avg": 0.182037353515625, + "step": 1134, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 236, + "val/ratio": 0.9879450798034668, + "val/ratio_var": 5.012234760215506e-05 + }, + { + "episode": 18160, + "epoch": 0.9171717171717172, + "eps": 0, + "loss/policy_avg": 0.06688633561134338, + "loss/value_avg": 0.4891657829284668, + "lr": 1.9173281579481896e-07, + "objective/entropy": -23.645580291748047, + "objective/kl": 38.17439270019531, + "objective/non_score_reward": -1.9087196588516235, + "objective/rlhf_reward": 15.474092483520508, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.000994181027635932, + "policy/clipfrac_avg": 0.0035517350770533085, + "policy/entropy_avg": 0.21059799194335938, + "step": 1135, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4, + "val/ratio": 0.9864646196365356, + "val/ratio_var": 6.11128198215738e-05 + }, + { + "episode": 18176, + "epoch": 0.917979797979798, + "eps": 0, + "loss/policy_avg": 0.09911377727985382, + "loss/value_avg": 0.18598923087120056, + "lr": 1.8808655548019638e-07, + "objective/entropy": -43.90042495727539, + "objective/kl": 43.086509704589844, + "objective/non_score_reward": -2.154325485229492, + "objective/rlhf_reward": 14.970674514770508, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0010021792259067297, + "policy/clipfrac_avg": 0.00640224339440465, + "policy/entropy_avg": 0.19201087951660156, + "step": 1136, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 455, + "val/ratio": 0.9868404865264893, + "val/ratio_var": 3.5931021557189524e-05 + }, + { + "episode": 18192, + "epoch": 0.9187878787878788, + "eps": 0, + "loss/policy_avg": -0.0433524064719677, + "loss/value_avg": 0.6105636358261108, + "lr": 1.8447463535214872e-07, + "objective/entropy": -149.0535125732422, + "objective/kl": 41.71107482910156, + "objective/non_score_reward": -2.0855536460876465, + "objective/rlhf_reward": 15.117570877075195, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0006844973540864885, + "policy/clipfrac_avg": 0.004837099928408861, + "policy/entropy_avg": 0.1427140235900879, + "step": 1137, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 356, + "val/ratio": 0.990210235118866, + "val/ratio_var": 5.7032579206861556e-05 + }, + { + "episode": 18208, + "epoch": 0.9195959595959596, + "eps": 0, + "loss/policy_avg": -0.038881611078977585, + "loss/value_avg": 0.2666724920272827, + "lr": 1.8089708118711525e-07, + "objective/entropy": -55.38002395629883, + "objective/kl": 38.607025146484375, + "objective/non_score_reward": -1.9303512573242188, + "objective/rlhf_reward": 14.397773742675781, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009189241100102663, + "policy/clipfrac_avg": 0.003879917087033391, + "policy/entropy_avg": 0.15883827209472656, + "step": 1138, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 41, + "val/ratio": 0.9893890619277954, + "val/ratio_var": 5.718938700738363e-05 + }, + { + "episode": 18224, + "epoch": 0.9204040404040404, + "eps": 0, + "loss/policy_avg": 0.06429219990968704, + "loss/value_avg": 0.8066933155059814, + "lr": 1.7735391851628814e-07, + "objective/entropy": -44.06977462768555, + "objective/kl": 34.568389892578125, + "objective/non_score_reward": -1.728419542312622, + "objective/rlhf_reward": 14.62314224243164, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008048653253354132, + "policy/clipfrac_avg": 0.006366053596138954, + "policy/entropy_avg": 0.18921661376953125, + "step": 1139, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 33, + "val/ratio": 0.9889757037162781, + "val/ratio_var": 3.707074938574806e-05 + }, + { + "episode": 18240, + "epoch": 0.9212121212121213, + "eps": 0, + "loss/policy_avg": 0.14161835610866547, + "loss/value_avg": 0.728713870048523, + "lr": 1.7384517262542255e-07, + "objective/entropy": -85.9615707397461, + "objective/kl": 39.761756896972656, + "objective/non_score_reward": -1.988087773323059, + "objective/rlhf_reward": 14.47284984588623, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009051311062648892, + "policy/clipfrac_avg": 0.0030989747028797865, + "policy/entropy_avg": 0.15839195251464844, + "step": 1140, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 59, + "val/ratio": 0.9888681173324585, + "val/ratio_var": 7.059999916236848e-05 + }, + { + "episode": 18256, + "epoch": 0.9220202020202021, + "eps": 0, + "loss/policy_avg": 0.10231739282608032, + "loss/value_avg": 0.7710623145103455, + "lr": 1.7037086855465902e-07, + "objective/entropy": -50.15681457519531, + "objective/kl": 42.769935607910156, + "objective/non_score_reward": -2.1384968757629395, + "objective/rlhf_reward": 14.103690147399902, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009481002343818545, + "policy/clipfrac_avg": 0.0051858918741345406, + "policy/entropy_avg": 0.2104654312133789, + "step": 1141, + "val/clipfrac_avg": 2.8775322789442725e-05, + "val/num_eos_tokens": 158, + "val/ratio": 0.9877794981002808, + "val/ratio_var": 6.828846380813047e-05 + }, + { + "episode": 18272, + "epoch": 0.9228282828282828, + "eps": 0, + "loss/policy_avg": 0.06746610999107361, + "loss/value_avg": 0.32568371295928955, + "lr": 1.6693103109834408e-07, + "objective/entropy": -83.49034118652344, + "objective/kl": 37.26749801635742, + "objective/non_score_reward": -1.863374948501587, + "objective/rlhf_reward": 13.496000289916992, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008203347679227591, + "policy/clipfrac_avg": 0.007063254248350859, + "policy/entropy_avg": 0.16310882568359375, + "step": 1142, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 109, + "val/ratio": 0.9898014068603516, + "val/ratio_var": 4.5457767555490136e-05 + }, + { + "episode": 18288, + "epoch": 0.9236363636363636, + "eps": 0, + "loss/policy_avg": -0.1702660322189331, + "loss/value_avg": 0.35972562432289124, + "lr": 1.6352568480485277e-07, + "objective/entropy": -22.768522262573242, + "objective/kl": 45.40270233154297, + "objective/non_score_reward": -2.2701351642608643, + "objective/rlhf_reward": 14.222051620483398, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.001096026157028973, + "policy/clipfrac_avg": 0.004881397821009159, + "policy/entropy_avg": 0.21602249145507812, + "step": 1143, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 105, + "val/ratio": 0.9855197668075562, + "val/ratio_var": 3.710871169460006e-05 + }, + { + "episode": 18304, + "epoch": 0.9244444444444444, + "eps": 0, + "loss/policy_avg": 0.19920386373996735, + "loss/value_avg": 0.5300213098526001, + "lr": 1.6015485397641696e-07, + "objective/entropy": -49.97838592529297, + "objective/kl": 33.057132720947266, + "objective/non_score_reward": -1.6528568267822266, + "objective/rlhf_reward": 15.800268173217773, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0009639778290875256, + "policy/clipfrac_avg": 0.00445273332297802, + "policy/entropy_avg": 0.180816650390625, + "step": 1144, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 508, + "val/ratio": 0.9873702526092529, + "val/ratio_var": 4.910973075311631e-05 + }, + { + "episode": 18320, + "epoch": 0.9252525252525252, + "eps": 0, + "loss/policy_avg": -0.09338352829217911, + "loss/value_avg": 0.23753158748149872, + "lr": 1.5681856266894612e-07, + "objective/entropy": -32.52317810058594, + "objective/kl": 46.23842239379883, + "objective/non_score_reward": -2.3119211196899414, + "objective/rlhf_reward": 14.336517333984375, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009981514886021614, + "policy/clipfrac_avg": 0.002960982732474804, + "policy/entropy_avg": 0.20490646362304688, + "step": 1145, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 284, + "val/ratio": 0.9861640334129333, + "val/ratio_var": 5.2834289817838e-05 + }, + { + "episode": 18336, + "epoch": 0.926060606060606, + "eps": 0, + "loss/policy_avg": -0.194179505109787, + "loss/value_avg": 0.8305935859680176, + "lr": 1.5351683469185973e-07, + "objective/entropy": -102.07046508789062, + "objective/kl": 31.8308162689209, + "objective/non_score_reward": -1.5915409326553345, + "objective/rlhf_reward": 14.752208709716797, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008943321881815791, + "policy/clipfrac_avg": 0.006945772562175989, + "policy/entropy_avg": 0.16139507293701172, + "step": 1146, + "val/clipfrac_avg": 0.00017361111531499773, + "val/num_eos_tokens": 780, + "val/ratio": 0.9883906245231628, + "val/ratio_var": 7.156646461226046e-05 + }, + { + "episode": 18352, + "epoch": 0.9268686868686868, + "eps": 0, + "loss/policy_avg": -0.14431902766227722, + "loss/value_avg": 1.0912277698516846, + "lr": 1.5024969360791564e-07, + "objective/entropy": -73.28153991699219, + "objective/kl": 31.00863265991211, + "objective/non_score_reward": -1.550431728363037, + "objective/rlhf_reward": 13.941755294799805, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000826484989374876, + "policy/clipfrac_avg": 0.004950974136590958, + "policy/entropy_avg": 0.15498733520507812, + "step": 1147, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 441, + "val/ratio": 0.9889597296714783, + "val/ratio_var": 6.787945312680677e-05 + }, + { + "episode": 18368, + "epoch": 0.9276767676767677, + "eps": 0, + "loss/policy_avg": -0.03664656728506088, + "loss/value_avg": 0.5489372611045837, + "lr": 1.4701716273304524e-07, + "objective/entropy": -56.737247467041016, + "objective/kl": 37.2110710144043, + "objective/non_score_reward": -1.860553503036499, + "objective/rlhf_reward": 13.061321258544922, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009301225654780865, + "policy/clipfrac_avg": 0.003762050997465849, + "policy/entropy_avg": 0.16833877563476562, + "step": 1148, + "val/clipfrac_avg": 0.010529030114412308, + "val/num_eos_tokens": 126, + "val/ratio": 0.9884514808654785, + "val/ratio_var": 3.90233144571539e-05 + }, + { + "episode": 18384, + "epoch": 0.9284848484848485, + "eps": 0, + "loss/policy_avg": -0.13550132513046265, + "loss/value_avg": 0.310035765171051, + "lr": 1.4381926513618139e-07, + "objective/entropy": -37.70539855957031, + "objective/kl": 41.92686462402344, + "objective/non_score_reward": -2.0963432788848877, + "objective/rlhf_reward": 14.622406959533691, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010944223031401634, + "policy/clipfrac_avg": 0.004749140702188015, + "policy/entropy_avg": 0.21954727172851562, + "step": 1149, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 114, + "val/ratio": 0.9851934313774109, + "val/ratio_var": 6.719871453242376e-05 + }, + { + "episode": 18400, + "epoch": 0.9292929292929293, + "eps": 0, + "loss/policy_avg": -0.06519678980112076, + "loss/value_avg": 0.4952123761177063, + "lr": 1.4065602363909747e-07, + "objective/entropy": -55.67281723022461, + "objective/kl": 38.189300537109375, + "objective/non_score_reward": -1.9094650745391846, + "objective/rlhf_reward": 14.028034210205078, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001011414104141295, + "policy/clipfrac_avg": 0.005307096056640148, + "policy/entropy_avg": 0.20238876342773438, + "step": 1150, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 103, + "val/ratio": 0.9863063097000122, + "val/ratio_var": 0.00010381833271821961 + }, + { + "episode": 18416, + "epoch": 0.9301010101010101, + "eps": 0, + "loss/policy_avg": 0.025855571031570435, + "loss/value_avg": 0.7742699980735779, + "lr": 1.375274608162447e-07, + "objective/entropy": -64.18814086914062, + "objective/kl": 39.29536056518555, + "objective/non_score_reward": -1.9647679328918457, + "objective/rlhf_reward": 13.746169090270996, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009069567313417792, + "policy/clipfrac_avg": 0.00267383293248713, + "policy/entropy_avg": 0.19156980514526367, + "step": 1151, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885040521621704, + "val/ratio_var": 7.799283775966614e-05 + }, + { + "episode": 18432, + "epoch": 0.9309090909090909, + "eps": 0, + "loss/policy_avg": 0.01431315392255783, + "loss/value_avg": 0.6286057829856873, + "lr": 1.3443359899458997e-07, + "objective/entropy": -59.235267639160156, + "objective/kl": 40.618900299072266, + "objective/non_score_reward": -2.030945062637329, + "objective/rlhf_reward": 11.54718017578125, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0008881621761247516, + "policy/clipfrac_avg": 0.0032237274572253227, + "policy/entropy_avg": 0.17874622344970703, + "step": 1152, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 353, + "val/ratio": 0.9887373447418213, + "val/ratio_var": 5.042510019848123e-05 + }, + { + "episode": 18448, + "epoch": 0.9317171717171717, + "eps": 0, + "loss/policy_avg": -0.0857156366109848, + "loss/value_avg": 0.2431882619857788, + "lr": 1.3137446025345557e-07, + "objective/entropy": -56.33277130126953, + "objective/kl": 46.32222366333008, + "objective/non_score_reward": -2.3161110877990723, + "objective/rlhf_reward": 15.996389389038086, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0009288631845265627, + "policy/clipfrac_avg": 0.0027455075178295374, + "policy/entropy_avg": 0.1957530975341797, + "step": 1153, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 26, + "val/ratio": 0.9869426488876343, + "val/ratio_var": 4.2594107071636245e-05 + }, + { + "episode": 18464, + "epoch": 0.9325252525252525, + "eps": 0, + "loss/policy_avg": -0.09154313802719116, + "loss/value_avg": 0.5833359360694885, + "lr": 1.283500664243642e-07, + "objective/entropy": -81.86225891113281, + "objective/kl": 33.21459197998047, + "objective/non_score_reward": -1.6607296466827393, + "objective/rlhf_reward": 13.62833309173584, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008894167840480804, + "policy/clipfrac_avg": 0.0032217884436249733, + "policy/entropy_avg": 0.16341471672058105, + "step": 1154, + "val/clipfrac_avg": 0.0005091222701594234, + "val/num_eos_tokens": 503, + "val/ratio": 0.9880057573318481, + "val/ratio_var": 7.092337909853086e-05 + }, + { + "episode": 18480, + "epoch": 0.9333333333333333, + "eps": 0, + "loss/policy_avg": 0.10637003928422928, + "loss/value_avg": 0.5488271713256836, + "lr": 1.253604390908819e-07, + "objective/entropy": -86.53608703613281, + "objective/kl": 37.38941192626953, + "objective/non_score_reward": -1.8694705963134766, + "objective/rlhf_reward": 14.591466903686523, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010367620270699263, + "policy/clipfrac_avg": 0.007691029459238052, + "policy/entropy_avg": 0.16927146911621094, + "step": 1155, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 399, + "val/ratio": 0.9881411790847778, + "val/ratio_var": 8.204267214750871e-05 + }, + { + "episode": 18496, + "epoch": 0.9341414141414142, + "eps": 0, + "loss/policy_avg": -0.024129033088684082, + "loss/value_avg": 0.4214504361152649, + "lr": 1.2240559958846266e-07, + "objective/entropy": -87.75643920898438, + "objective/kl": 44.4080810546875, + "objective/non_score_reward": -2.2204041481018066, + "objective/rlhf_reward": 14.404596328735352, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008544453303329647, + "policy/clipfrac_avg": 0.0035026988480240107, + "policy/entropy_avg": 0.16388320922851562, + "step": 1156, + "val/clipfrac_avg": 9.057971328729764e-05, + "val/num_eos_tokens": 8, + "val/ratio": 0.9889931082725525, + "val/ratio_var": 6.746850704075769e-05 + }, + { + "episode": 18512, + "epoch": 0.934949494949495, + "eps": 0, + "loss/policy_avg": -0.08957675099372864, + "loss/value_avg": 0.3355456590652466, + "lr": 1.1948556900429786e-07, + "objective/entropy": -90.10191345214844, + "objective/kl": 39.96202850341797, + "objective/non_score_reward": -1.9981014728546143, + "objective/rlhf_reward": 16.14252471923828, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008546580211259425, + "policy/clipfrac_avg": 0.005381043069064617, + "policy/entropy_avg": 0.16065406799316406, + "step": 1157, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3, + "val/ratio": 0.989165186882019, + "val/ratio_var": 4.7844900109339505e-05 + }, + { + "episode": 18528, + "epoch": 0.9357575757575758, + "eps": 0, + "loss/policy_avg": -0.06473330408334732, + "loss/value_avg": 0.4225938320159912, + "lr": 1.1660036817716492e-07, + "objective/entropy": -63.458831787109375, + "objective/kl": 40.30827331542969, + "objective/non_score_reward": -2.015413761138916, + "objective/rlhf_reward": 13.633024215698242, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008652628748677671, + "policy/clipfrac_avg": 0.009679234586656094, + "policy/entropy_avg": 0.1683187484741211, + "step": 1158, + "val/clipfrac_avg": 0.000117481205961667, + "val/num_eos_tokens": 301, + "val/ratio": 0.989500880241394, + "val/ratio_var": 3.429090429563075e-05 + }, + { + "episode": 18544, + "epoch": 0.9365656565656566, + "eps": 0, + "loss/policy_avg": 0.19546493887901306, + "loss/value_avg": 0.6905479431152344, + "lr": 1.1375001769728e-07, + "objective/entropy": -65.75634765625, + "objective/kl": 36.45595169067383, + "objective/non_score_reward": -1.8227976560592651, + "objective/rlhf_reward": 13.138139724731445, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008474080823361874, + "policy/clipfrac_avg": 0.006815841421484947, + "policy/entropy_avg": 0.16717529296875, + "step": 1159, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 124, + "val/ratio": 0.9894546270370483, + "val/ratio_var": 7.423999340971932e-05 + }, + { + "episode": 18560, + "epoch": 0.9373737373737374, + "eps": 0, + "loss/policy_avg": 0.10427418351173401, + "loss/value_avg": 0.7115825414657593, + "lr": 1.1093453790614883e-07, + "objective/entropy": -168.40960693359375, + "objective/kl": 37.055477142333984, + "objective/non_score_reward": -1.852773666381836, + "objective/rlhf_reward": 14.084726333618164, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0007912840810604393, + "policy/clipfrac_avg": 0.0036162808537483215, + "policy/entropy_avg": 0.1563091278076172, + "step": 1160, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 427, + "val/ratio": 0.9901214838027954, + "val/ratio_var": 7.212928176159039e-05 + }, + { + "episode": 18576, + "epoch": 0.9381818181818182, + "eps": 0, + "loss/policy_avg": -0.12785540521144867, + "loss/value_avg": 0.39417845010757446, + "lr": 1.0815394889642339e-07, + "objective/entropy": -70.233154296875, + "objective/kl": 44.201019287109375, + "objective/non_score_reward": -2.2100508213043213, + "objective/rlhf_reward": 16.102449417114258, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011408152058720589, + "policy/clipfrac_avg": 0.004944411106407642, + "policy/entropy_avg": 0.19538497924804688, + "step": 1161, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 331, + "val/ratio": 0.9865884184837341, + "val/ratio_var": 6.0507758462335914e-05 + }, + { + "episode": 18592, + "epoch": 0.938989898989899, + "eps": 0, + "loss/policy_avg": -0.03844344615936279, + "loss/value_avg": 0.655718207359314, + "lr": 1.0540827051175817e-07, + "objective/entropy": -37.648441314697266, + "objective/kl": 43.3171501159668, + "objective/non_score_reward": -2.1658573150634766, + "objective/rlhf_reward": 13.099767684936523, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009900203440338373, + "policy/clipfrac_avg": 0.003007909283041954, + "policy/entropy_avg": 0.20216941833496094, + "step": 1162, + "val/clipfrac_avg": 0.0003906250058207661, + "val/num_eos_tokens": 199, + "val/ratio": 0.9874058961868286, + "val/ratio_var": 6.270931771723554e-05 + }, + { + "episode": 18608, + "epoch": 0.9397979797979797, + "eps": 0, + "loss/policy_avg": 0.05490083992481232, + "loss/value_avg": 0.4446883201599121, + "lr": 1.0269752234666642e-07, + "objective/entropy": -83.78005981445312, + "objective/kl": 38.83243942260742, + "objective/non_score_reward": -1.941622018814087, + "objective/rlhf_reward": 15.863064765930176, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008689056267030537, + "policy/clipfrac_avg": 0.004589558579027653, + "policy/entropy_avg": 0.1755847930908203, + "step": 1163, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 209, + "val/ratio": 0.9878473281860352, + "val/ratio_var": 5.755285747000016e-05 + }, + { + "episode": 18624, + "epoch": 0.9406060606060606, + "eps": 0, + "loss/policy_avg": 0.5652229189872742, + "loss/value_avg": 4.454827785491943, + "lr": 1.0002172374638519e-07, + "objective/entropy": -106.71968078613281, + "objective/kl": 28.182273864746094, + "objective/non_score_reward": -1.4091136455535889, + "objective/rlhf_reward": 13.051824569702148, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006604374502785504, + "policy/clipfrac_avg": 0.006071992218494415, + "policy/entropy_avg": 0.13239336013793945, + "step": 1164, + "val/clipfrac_avg": 0.0032051282469183207, + "val/num_eos_tokens": 199, + "val/ratio": 0.9910687208175659, + "val/ratio_var": 5.329243867890909e-05 + }, + { + "episode": 18640, + "epoch": 0.9414141414141414, + "eps": 0, + "loss/policy_avg": 0.030554044991731644, + "loss/value_avg": 0.3813501298427582, + "lr": 9.738089380673e-08, + "objective/entropy": -50.39229965209961, + "objective/kl": 46.05052947998047, + "objective/non_score_reward": -2.3025264739990234, + "objective/rlhf_reward": 15.775598526000977, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0010013967985287309, + "policy/clipfrac_avg": 0.004134259186685085, + "policy/entropy_avg": 0.20552825927734375, + "step": 1165, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 147, + "val/ratio": 0.986208975315094, + "val/ratio_var": 5.7627206842880696e-05 + }, + { + "episode": 18656, + "epoch": 0.9422222222222222, + "eps": 0, + "loss/policy_avg": 0.014425485394895077, + "loss/value_avg": 0.17465442419052124, + "lr": 9.477505137396759e-08, + "objective/entropy": -48.5864143371582, + "objective/kl": 46.88139343261719, + "objective/non_score_reward": -2.344069719314575, + "objective/rlhf_reward": 14.749679565429688, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009588432731106877, + "policy/clipfrac_avg": 0.003158081788569689, + "policy/entropy_avg": 0.19722747802734375, + "step": 1166, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 150, + "val/ratio": 0.9872362613677979, + "val/ratio_var": 5.964138108538464e-05 + }, + { + "episode": 18672, + "epoch": 0.943030303030303, + "eps": 0, + "loss/policy_avg": 0.12084536254405975, + "loss/value_avg": 0.5831397771835327, + "lr": 9.22042150446728e-08, + "objective/entropy": -76.38983154296875, + "objective/kl": 38.01190948486328, + "objective/non_score_reward": -1.9005954265594482, + "objective/rlhf_reward": 11.919716835021973, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0007046982645988464, + "policy/clipfrac_avg": 0.0028958613984286785, + "policy/entropy_avg": 0.1511383056640625, + "step": 1167, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 417, + "val/ratio": 0.990734875202179, + "val/ratio_var": 6.113814743002877e-05 + }, + { + "episode": 18688, + "epoch": 0.9438383838383838, + "eps": 0, + "loss/policy_avg": -0.1721150279045105, + "loss/value_avg": 0.29965418577194214, + "lr": 8.966840316560255e-08, + "objective/entropy": -190.17677307128906, + "objective/kl": 44.538299560546875, + "objective/non_score_reward": -2.226915121078491, + "objective/rlhf_reward": 14.882460594177246, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0007411800906993449, + "policy/clipfrac_avg": 0.005143396090716124, + "policy/entropy_avg": 0.14809703826904297, + "step": 1168, + "val/clipfrac_avg": 1.3469827536027879e-05, + "val/num_eos_tokens": 76, + "val/ratio": 0.990103542804718, + "val/ratio_var": 4.799567614099942e-05 + }, + { + "episode": 18704, + "epoch": 0.9446464646464646, + "eps": 0, + "loss/policy_avg": -0.04706333950161934, + "loss/value_avg": 0.4846585988998413, + "lr": 8.716763383355863e-08, + "objective/entropy": -26.354211807250977, + "objective/kl": 43.76121520996094, + "objective/non_score_reward": -2.1880605220794678, + "objective/rlhf_reward": 12.905689239501953, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010139367077499628, + "policy/clipfrac_avg": 0.00270761433057487, + "policy/entropy_avg": 0.19493675231933594, + "step": 1169, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 515, + "val/ratio": 0.9869682788848877, + "val/ratio_var": 4.458573312149383e-05 + }, + { + "episode": 18720, + "epoch": 0.9454545454545454, + "eps": 0, + "loss/policy_avg": -0.11434232443571091, + "loss/value_avg": 0.32796037197113037, + "lr": 8.470192489526519e-08, + "objective/entropy": -118.44176483154297, + "objective/kl": 33.78557205200195, + "objective/non_score_reward": -1.6892786026000977, + "objective/rlhf_reward": 14.466971397399902, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008025875431485474, + "policy/clipfrac_avg": 0.005034630186855793, + "policy/entropy_avg": 0.17252540588378906, + "step": 1170, + "val/clipfrac_avg": 0.000240384615608491, + "val/num_eos_tokens": 534, + "val/ratio": 0.9892752766609192, + "val/ratio_var": 8.975582750281319e-05 + }, + { + "episode": 18736, + "epoch": 0.9462626262626262, + "eps": 0, + "loss/policy_avg": -0.26330432295799255, + "loss/value_avg": 0.43789923191070557, + "lr": 8.227129394723643e-08, + "objective/entropy": -90.45104217529297, + "objective/kl": 35.37781524658203, + "objective/non_score_reward": -1.7688908576965332, + "objective/rlhf_reward": 15.840483665466309, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0007532388553954661, + "policy/clipfrac_avg": 0.00782378576695919, + "policy/entropy_avg": 0.14811134338378906, + "step": 1171, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 449, + "val/ratio": 0.9901248216629028, + "val/ratio_var": 4.5044969738228247e-05 + }, + { + "episode": 18752, + "epoch": 0.9470707070707071, + "eps": 0, + "loss/policy_avg": 0.09692054986953735, + "loss/value_avg": 0.2597772777080536, + "lr": 7.987575833565298e-08, + "objective/entropy": -90.80900573730469, + "objective/kl": 45.11640930175781, + "objective/non_score_reward": -2.2558202743530273, + "objective/rlhf_reward": 13.744178771972656, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000919578829780221, + "policy/clipfrac_avg": 0.005172869190573692, + "policy/entropy_avg": 0.17873764038085938, + "step": 1172, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 400, + "val/ratio": 0.9880226850509644, + "val/ratio_var": 5.284639337332919e-05 + }, + { + "episode": 18768, + "epoch": 0.9478787878787879, + "eps": 0, + "loss/policy_avg": 0.09063289314508438, + "loss/value_avg": 0.5753982663154602, + "lr": 7.7515335156238e-08, + "objective/entropy": -163.6809844970703, + "objective/kl": 37.627655029296875, + "objective/non_score_reward": -1.8813828229904175, + "objective/rlhf_reward": 14.59518051147461, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0007881260244175792, + "policy/clipfrac_avg": 0.0037529754918068647, + "policy/entropy_avg": 0.15079402923583984, + "step": 1173, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 262, + "val/ratio": 0.9891691207885742, + "val/ratio_var": 4.976944182999432e-05 + }, + { + "episode": 18784, + "epoch": 0.9486868686868687, + "eps": 0, + "loss/policy_avg": 0.14138484001159668, + "loss/value_avg": 0.9088690280914307, + "lr": 7.519004125413454e-08, + "objective/entropy": -171.2418670654297, + "objective/kl": 38.040130615234375, + "objective/non_score_reward": -1.9020065069198608, + "objective/rlhf_reward": 13.176118850708008, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0006656637415289879, + "policy/clipfrac_avg": 0.0033770238514989614, + "policy/entropy_avg": 0.1490926742553711, + "step": 1174, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 584, + "val/ratio": 0.9910387992858887, + "val/ratio_var": 5.571794099523686e-05 + }, + { + "episode": 18800, + "epoch": 0.9494949494949495, + "eps": 0, + "loss/policy_avg": -0.09168200194835663, + "loss/value_avg": 0.44036585092544556, + "lr": 7.289989322378732e-08, + "objective/entropy": -50.938758850097656, + "objective/kl": 43.64716339111328, + "objective/non_score_reward": -2.1823582649230957, + "objective/rlhf_reward": 13.802017211914062, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009952617110684514, + "policy/clipfrac_avg": 0.0027715780306607485, + "policy/entropy_avg": 0.17894935607910156, + "step": 1175, + "val/clipfrac_avg": 0.0004111842135898769, + "val/num_eos_tokens": 169, + "val/ratio": 0.987088680267334, + "val/ratio_var": 6.252353341551498e-05 + }, + { + "episode": 18816, + "epoch": 0.9503030303030303, + "eps": 0, + "loss/policy_avg": -0.17822237312793732, + "loss/value_avg": 0.4382031559944153, + "lr": 7.064490740882057e-08, + "objective/entropy": -153.3197479248047, + "objective/kl": 43.219486236572266, + "objective/non_score_reward": -2.1609742641448975, + "objective/rlhf_reward": 12.741369247436523, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009114840067923069, + "policy/clipfrac_avg": 0.004199516959488392, + "policy/entropy_avg": 0.18891334533691406, + "step": 1176, + "val/clipfrac_avg": 3.814697265625e-05, + "val/num_eos_tokens": 23, + "val/ratio": 0.9868748784065247, + "val/ratio_var": 7.712288061156869e-05 + }, + { + "episode": 18832, + "epoch": 0.9511111111111111, + "eps": 0, + "loss/policy_avg": -0.29926538467407227, + "loss/value_avg": 0.24289564788341522, + "lr": 6.842509990192536e-08, + "objective/entropy": -62.70560073852539, + "objective/kl": 46.933937072753906, + "objective/non_score_reward": -2.3466968536376953, + "objective/rlhf_reward": 15.286115646362305, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0009256636258214712, + "policy/clipfrac_avg": 0.005345791578292847, + "policy/entropy_avg": 0.19078445434570312, + "step": 1177, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878906607627869, + "val/ratio_var": 5.454176425701007e-05 + }, + { + "episode": 18848, + "epoch": 0.9519191919191919, + "eps": 0, + "loss/policy_avg": -0.07624176144599915, + "loss/value_avg": 1.0276535749435425, + "lr": 6.624048654474247e-08, + "objective/entropy": -102.2330322265625, + "objective/kl": 38.56993103027344, + "objective/non_score_reward": -1.9284967184066772, + "objective/rlhf_reward": 13.352752685546875, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008128330809995532, + "policy/clipfrac_avg": 0.004654319025576115, + "policy/entropy_avg": 0.16016387939453125, + "step": 1178, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 33, + "val/ratio": 0.9896093010902405, + "val/ratio_var": 5.181640881346539e-05 + }, + { + "episode": 18864, + "epoch": 0.9527272727272728, + "eps": 0, + "loss/policy_avg": 0.035814136266708374, + "loss/value_avg": 0.5764297842979431, + "lr": 6.409108292774912e-08, + "objective/entropy": -59.69445037841797, + "objective/kl": 39.02704620361328, + "objective/non_score_reward": -1.9513523578643799, + "objective/rlhf_reward": 13.173646926879883, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001039088936522603, + "policy/clipfrac_avg": 0.003585438709706068, + "policy/entropy_avg": 0.19232654571533203, + "step": 1179, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 192, + "val/ratio": 0.9868942499160767, + "val/ratio_var": 6.958071753615513e-05 + }, + { + "episode": 18880, + "epoch": 0.9535353535353536, + "eps": 0, + "loss/policy_avg": 0.24797323346138, + "loss/value_avg": 0.2845846712589264, + "lr": 6.197690439015025e-08, + "objective/entropy": -40.500762939453125, + "objective/kl": 47.32844543457031, + "objective/non_score_reward": -2.366422176361084, + "objective/rlhf_reward": 13.852327346801758, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010082677472382784, + "policy/clipfrac_avg": 0.005784696899354458, + "policy/entropy_avg": 0.19600677490234375, + "step": 1180, + "val/clipfrac_avg": 0.00013241525448393077, + "val/num_eos_tokens": 504, + "val/ratio": 0.9869298934936523, + "val/ratio_var": 4.6343844587681815e-05 + }, + { + "episode": 18896, + "epoch": 0.9543434343434344, + "eps": 0, + "loss/policy_avg": -0.011056587100028992, + "loss/value_avg": 0.21195590496063232, + "lr": 5.989796601976516e-08, + "objective/entropy": -87.35359191894531, + "objective/kl": 38.190425872802734, + "objective/non_score_reward": -1.9095213413238525, + "objective/rlhf_reward": 12.574853897094727, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000693869951646775, + "policy/clipfrac_avg": 0.0022745702881366014, + "policy/entropy_avg": 0.14525604248046875, + "step": 1181, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 672, + "val/ratio": 0.9902079105377197, + "val/ratio_var": 5.0829687097575516e-05 + }, + { + "episode": 18912, + "epoch": 0.9551515151515152, + "eps": 0, + "loss/policy_avg": 0.5624556541442871, + "loss/value_avg": 0.936620831489563, + "lr": 5.785428265292381e-08, + "objective/entropy": -131.93319702148438, + "objective/kl": 36.28763961791992, + "objective/non_score_reward": -1.8143819570541382, + "objective/rlhf_reward": 12.099679946899414, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0006996375159360468, + "policy/clipfrac_avg": 0.0033093849197030067, + "policy/entropy_avg": 0.1407628059387207, + "step": 1182, + "val/clipfrac_avg": 0.0014880952658131719, + "val/num_eos_tokens": 240, + "val/ratio": 0.9909154176712036, + "val/ratio_var": 5.887184670427814e-05 + }, + { + "episode": 18928, + "epoch": 0.955959595959596, + "eps": 0, + "loss/policy_avg": -0.19112437963485718, + "loss/value_avg": 0.6247477531433105, + "lr": 5.584586887435739e-08, + "objective/entropy": -117.6522216796875, + "objective/kl": 39.66671371459961, + "objective/non_score_reward": -1.9833356142044067, + "objective/rlhf_reward": 13.297914505004883, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007932945154607296, + "policy/clipfrac_avg": 0.0029290905222296715, + "policy/entropy_avg": 0.16626358032226562, + "step": 1183, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 41, + "val/ratio": 0.9893531799316406, + "val/ratio_var": 5.722875357605517e-05 + }, + { + "episode": 18944, + "epoch": 0.9567676767676768, + "eps": 0, + "loss/policy_avg": -0.018872268497943878, + "loss/value_avg": 0.6814364194869995, + "lr": 5.387273901709733e-08, + "objective/entropy": -59.193870544433594, + "objective/kl": 38.85816955566406, + "objective/non_score_reward": -1.942908525466919, + "objective/rlhf_reward": 12.28365421295166, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009058800060302019, + "policy/clipfrac_avg": 0.004390705842524767, + "policy/entropy_avg": 0.17277050018310547, + "step": 1184, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 391, + "val/ratio": 0.9888806939125061, + "val/ratio_var": 5.870430322829634e-05 + }, + { + "episode": 18960, + "epoch": 0.9575757575757575, + "eps": 0, + "loss/policy_avg": -0.12193582952022552, + "loss/value_avg": 0.4072027802467346, + "lr": 5.1934907162370374e-08, + "objective/entropy": -72.95974731445312, + "objective/kl": 35.59428405761719, + "objective/non_score_reward": -1.7797143459320068, + "objective/rlhf_reward": 13.673410415649414, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008201617165468633, + "policy/clipfrac_avg": 0.004145351238548756, + "policy/entropy_avg": 0.17438912391662598, + "step": 1185, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 60, + "val/ratio": 0.9888736605644226, + "val/ratio_var": 6.133368151495233e-05 + }, + { + "episode": 18976, + "epoch": 0.9583838383838383, + "eps": 0, + "loss/policy_avg": 0.3409832715988159, + "loss/value_avg": 2.1178839206695557, + "lr": 5.0032387139501425e-08, + "objective/entropy": -97.86048889160156, + "objective/kl": 41.09333038330078, + "objective/non_score_reward": -2.054666519165039, + "objective/rlhf_reward": 13.710959434509277, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.000780026544816792, + "policy/clipfrac_avg": 0.007402490358799696, + "policy/entropy_avg": 0.19989013671875, + "step": 1186, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 76, + "val/ratio": 0.9890344738960266, + "val/ratio_var": 7.444098446285352e-05 + }, + { + "episode": 18992, + "epoch": 0.9591919191919192, + "eps": 0, + "loss/policy_avg": -0.16587065160274506, + "loss/value_avg": 0.5858635902404785, + "lr": 4.8165192525809754e-08, + "objective/entropy": -31.46831512451172, + "objective/kl": 36.85295486450195, + "objective/non_score_reward": -1.842647671699524, + "objective/rlhf_reward": 14.922977447509766, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.001208727015182376, + "policy/clipfrac_avg": 0.004555144812911749, + "policy/entropy_avg": 0.20450401306152344, + "step": 1187, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 113, + "val/ratio": 0.9874365329742432, + "val/ratio_var": 5.309836706146598e-05 + }, + { + "episode": 19008, + "epoch": 0.96, + "eps": 0, + "loss/policy_avg": 0.06852813065052032, + "loss/value_avg": 0.32328563928604126, + "lr": 4.63333366465174e-08, + "objective/entropy": -105.82904815673828, + "objective/kl": 32.94627380371094, + "objective/non_score_reward": -1.6473137140274048, + "objective/rlhf_reward": 15.290185928344727, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0006628286791965365, + "policy/clipfrac_avg": 0.004865474533289671, + "policy/entropy_avg": 0.12561321258544922, + "step": 1188, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 202, + "val/ratio": 0.9917237758636475, + "val/ratio_var": 3.8588947063544765e-05 + }, + { + "episode": 19024, + "epoch": 0.9608080808080808, + "eps": 0, + "loss/policy_avg": 0.020303204655647278, + "loss/value_avg": 0.4330519437789917, + "lr": 4.4536832574650355e-08, + "objective/entropy": -33.98373031616211, + "objective/kl": 45.81948471069336, + "objective/non_score_reward": -2.2909743785858154, + "objective/rlhf_reward": 13.005900382995605, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010197311639785767, + "policy/clipfrac_avg": 0.003383245086297393, + "policy/entropy_avg": 0.21395492553710938, + "step": 1189, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 86, + "val/ratio": 0.985183835029602, + "val/ratio_var": 5.527482062461786e-05 + }, + { + "episode": 19040, + "epoch": 0.9616161616161616, + "eps": 0, + "loss/policy_avg": -0.6133148074150085, + "loss/value_avg": 0.3876040279865265, + "lr": 4.2775693130948094e-08, + "objective/entropy": -152.57052612304688, + "objective/kl": 38.252220153808594, + "objective/non_score_reward": -1.9126112461090088, + "objective/rlhf_reward": 14.89988899230957, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007638135575689375, + "policy/clipfrac_avg": 0.0035591432824730873, + "policy/entropy_avg": 0.16634774208068848, + "step": 1190, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 392, + "val/ratio": 0.9892970323562622, + "val/ratio_var": 6.732134352205321e-05 + }, + { + "episode": 19056, + "epoch": 0.9624242424242424, + "eps": 0, + "loss/policy_avg": -0.16719648241996765, + "loss/value_avg": 0.4944162368774414, + "lr": 4.104993088376974e-08, + "objective/entropy": -81.75244140625, + "objective/kl": 46.120994567871094, + "objective/non_score_reward": -2.3060498237609863, + "objective/rlhf_reward": 13.537699699401855, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009323159465566278, + "policy/clipfrac_avg": 0.003273975569754839, + "policy/entropy_avg": 0.19600319862365723, + "step": 1191, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 291, + "val/ratio": 0.9876390695571899, + "val/ratio_var": 4.03721714974381e-05 + }, + { + "episode": 19072, + "epoch": 0.9632323232323232, + "eps": 0, + "loss/policy_avg": 0.08641587942838669, + "loss/value_avg": 0.6021695137023926, + "lr": 3.935955814900527e-08, + "objective/entropy": -58.236507415771484, + "objective/kl": 38.58598327636719, + "objective/non_score_reward": -1.9292993545532227, + "objective/rlhf_reward": 13.148825645446777, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0010641806293278933, + "policy/clipfrac_avg": 0.004020705819129944, + "policy/entropy_avg": 0.19188880920410156, + "step": 1192, + "val/clipfrac_avg": 0.00021150591783225536, + "val/num_eos_tokens": 665, + "val/ratio": 0.9875043630599976, + "val/ratio_var": 7.918935443740338e-05 + }, + { + "episode": 19088, + "epoch": 0.964040404040404, + "eps": 0, + "loss/policy_avg": 0.09802606701850891, + "loss/value_avg": 1.0540528297424316, + "lr": 3.77045869899878e-08, + "objective/entropy": -126.37530517578125, + "objective/kl": 36.086280822753906, + "objective/non_score_reward": -1.8043140172958374, + "objective/rlhf_reward": 13.906623840332031, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008227983489632607, + "policy/clipfrac_avg": 0.004573613405227661, + "policy/entropy_avg": 0.15534448623657227, + "step": 1193, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 325, + "val/ratio": 0.9889199733734131, + "val/ratio_var": 9.045511978911236e-05 + }, + { + "episode": 19104, + "epoch": 0.9648484848484848, + "eps": 0, + "loss/policy_avg": -0.17688646912574768, + "loss/value_avg": 0.4370724558830261, + "lr": 3.608502921740753e-08, + "objective/entropy": -35.11339569091797, + "objective/kl": 32.869319915771484, + "objective/non_score_reward": -1.6434661149978638, + "objective/rlhf_reward": 14.551846504211426, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.002314617158845067, + "policy/clipfrac_avg": 0.0031052969861775637, + "policy/entropy_avg": 0.17627334594726562, + "step": 1194, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 185, + "val/ratio": 0.989388108253479, + "val/ratio_var": 5.49936194147449e-05 + }, + { + "episode": 19120, + "epoch": 0.9656565656565657, + "eps": 0, + "loss/policy_avg": -0.17504875361919403, + "loss/value_avg": 0.204596608877182, + "lr": 3.450089638922738e-08, + "objective/entropy": -92.06233978271484, + "objective/kl": 40.820533752441406, + "objective/non_score_reward": -2.0410265922546387, + "objective/rlhf_reward": 14.513660430908203, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009747116710059345, + "policy/clipfrac_avg": 0.005887703038752079, + "policy/entropy_avg": 0.17911338806152344, + "step": 1195, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 294, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 4.2761934309965e-05 + }, + { + "episode": 19136, + "epoch": 0.9664646464646465, + "eps": 0, + "loss/policy_avg": -0.12304341048002243, + "loss/value_avg": 0.32093346118927, + "lr": 3.2952199810599714e-08, + "objective/entropy": -24.155014038085938, + "objective/kl": 45.62751007080078, + "objective/non_score_reward": -2.2813756465911865, + "objective/rlhf_reward": 14.195186614990234, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010361537570133805, + "policy/clipfrac_avg": 0.006471775472164154, + "policy/entropy_avg": 0.19795608520507812, + "step": 1196, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 209, + "val/ratio": 0.9868389368057251, + "val/ratio_var": 5.5822583817644045e-05 + }, + { + "episode": 19152, + "epoch": 0.9672727272727273, + "eps": 0, + "loss/policy_avg": 0.3443071246147156, + "loss/value_avg": 1.5372172594070435, + "lr": 3.143895053378698e-08, + "objective/entropy": -111.23834228515625, + "objective/kl": 31.232709884643555, + "objective/non_score_reward": -1.5616354942321777, + "objective/rlhf_reward": 13.35242748260498, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0007176597137004137, + "policy/clipfrac_avg": 0.004162969067692757, + "policy/entropy_avg": 0.2422410249710083, + "step": 1197, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 524, + "val/ratio": 0.9907166957855225, + "val/ratio_var": 7.462361827492714e-05 + }, + { + "episode": 19168, + "epoch": 0.9680808080808081, + "eps": 0, + "loss/policy_avg": -0.13895125687122345, + "loss/value_avg": 0.5284525752067566, + "lr": 2.9961159358082305e-08, + "objective/entropy": -101.44007110595703, + "objective/kl": 32.114402770996094, + "objective/non_score_reward": -1.6057199239730835, + "objective/rlhf_reward": 14.378654479980469, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000817152438685298, + "policy/clipfrac_avg": 0.004995419643819332, + "policy/entropy_avg": 0.15488815307617188, + "step": 1198, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 443, + "val/ratio": 0.9890648126602173, + "val/ratio_var": 7.766807539155707e-05 + }, + { + "episode": 19184, + "epoch": 0.9688888888888889, + "eps": 0, + "loss/policy_avg": -0.2223341315984726, + "loss/value_avg": 0.6924841403961182, + "lr": 2.8518836829732332e-08, + "objective/entropy": -71.81754302978516, + "objective/kl": 37.39568328857422, + "objective/non_score_reward": -1.8697842359542847, + "objective/rlhf_reward": 13.520840644836426, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008745577652007341, + "policy/clipfrac_avg": 0.002502672839909792, + "policy/entropy_avg": 0.18534183502197266, + "step": 1199, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 599, + "val/ratio": 0.9876028299331665, + "val/ratio_var": 5.801619045087136e-05 + }, + { + "episode": 19200, + "epoch": 0.9696969696969697, + "eps": 0, + "loss/policy_avg": 0.15578943490982056, + "loss/value_avg": 0.9613189101219177, + "lr": 2.7111993241860646e-08, + "objective/entropy": -74.75447082519531, + "objective/kl": 31.942792892456055, + "objective/non_score_reward": -1.597139596939087, + "objective/rlhf_reward": 13.043485641479492, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007055438472889364, + "policy/clipfrac_avg": 0.0035820305347442627, + "policy/entropy_avg": 0.15447449684143066, + "step": 1200, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 539, + "val/ratio": 0.9913029670715332, + "val/ratio_var": 5.023244739277288e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b1d8e7e2ac9ea6e4da4ba74ca10e6806c417f2d3 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46457f891d9640d8d717fa8b25fca4d200c9b233cc4a801c04d90dc7cfa790b6 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1ff9797fba0f6c56ef6e81f120ca1d35304e8dc6 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/trainer_state.json @@ -0,0 +1,26033 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 19808, + "epoch": 1.0004040404040404, + "eval_steps": 100, + "global_step": 1238, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + }, + { + "episode": 11216, + "epoch": 0.5664646464646464, + "eps": 0, + "loss/policy_avg": -0.09055965393781662, + "loss/value_avg": 0.5455071926116943, + "lr": 4.334128543718295e-06, + "objective/entropy": -74.95745849609375, + "objective/kl": 33.42132568359375, + "objective/non_score_reward": -1.6710662841796875, + "objective/rlhf_reward": 14.828933715820312, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0013184514828026295, + "policy/clipfrac_avg": 0.00809475127607584, + "policy/entropy_avg": 0.17902374267578125, + "step": 701, + "val/clipfrac_avg": 0.18833743035793304, + "val/num_eos_tokens": 358, + "val/ratio": 0.9879262447357178, + "val/ratio_var": 6.120221951277927e-05 + }, + { + "episode": 11232, + "epoch": 0.5672727272727273, + "eps": 0, + "loss/policy_avg": 0.2261451780796051, + "loss/value_avg": 1.5403046607971191, + "lr": 4.320892800523827e-06, + "objective/entropy": -90.61648559570312, + "objective/kl": 27.537485122680664, + "objective/non_score_reward": -1.3768744468688965, + "objective/rlhf_reward": 12.826250076293945, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009536159923300147, + "policy/clipfrac_avg": 0.01545755285769701, + "policy/entropy_avg": 0.20562219619750977, + "step": 702, + "val/clipfrac_avg": 0.033450622111558914, + "val/num_eos_tokens": 595, + "val/ratio": 0.9873440265655518, + "val/ratio_var": 7.613677007611841e-05 + }, + { + "episode": 11248, + "epoch": 0.5680808080808081, + "eps": 0, + "loss/policy_avg": -0.05153674632310867, + "loss/value_avg": 0.18954747915267944, + "lr": 4.307661903773129e-06, + "objective/entropy": -56.119659423828125, + "objective/kl": 43.216548919677734, + "objective/non_score_reward": -2.160827398300171, + "objective/rlhf_reward": 15.80792236328125, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010682737920433283, + "policy/clipfrac_avg": 0.005114596337080002, + "policy/entropy_avg": 0.20781707763671875, + "step": 703, + "val/clipfrac_avg": 0.12178023159503937, + "val/num_eos_tokens": 244, + "val/ratio": 0.9860707521438599, + "val/ratio_var": 5.392924140323885e-05 + }, + { + "episode": 11264, + "epoch": 0.5688888888888889, + "eps": 0, + "loss/policy_avg": 0.22105136513710022, + "loss/value_avg": 0.43452760577201843, + "lr": 4.294435947888405e-06, + "objective/entropy": -180.00924682617188, + "objective/kl": 32.27890396118164, + "objective/non_score_reward": -1.6139451265335083, + "objective/rlhf_reward": 12.425117492675781, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007389815291389823, + "policy/clipfrac_avg": 0.00613048393279314, + "policy/entropy_avg": 0.15887287259101868, + "step": 704, + "val/clipfrac_avg": 0.12273523956537247, + "val/num_eos_tokens": 611, + "val/ratio": 0.9900252819061279, + "val/ratio_var": 6.185931124491617e-05 + }, + { + "episode": 11280, + "epoch": 0.5696969696969697, + "eps": 0, + "loss/policy_avg": -0.044020622968673706, + "loss/value_avg": 1.459969401359558, + "lr": 4.281215027256592e-06, + "objective/entropy": -73.2503890991211, + "objective/kl": 29.207317352294922, + "objective/non_score_reward": -1.460365891456604, + "objective/rlhf_reward": 15.766195297241211, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010228066239506006, + "policy/clipfrac_avg": 0.010731351561844349, + "policy/entropy_avg": 0.17015647888183594, + "step": 705, + "val/clipfrac_avg": 0.1418685019016266, + "val/num_eos_tokens": 378, + "val/ratio": 0.9865360856056213, + "val/ratio_var": 0.0001155980207840912 + }, + { + "episode": 11296, + "epoch": 0.5705050505050505, + "eps": 0, + "loss/policy_avg": 0.10734500735998154, + "loss/value_avg": 0.40052735805511475, + "lr": 4.2679992362287e-06, + "objective/entropy": -91.05632019042969, + "objective/kl": 39.329261779785156, + "objective/non_score_reward": -1.966463327407837, + "objective/rlhf_reward": 12.744474411010742, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010600072564557195, + "policy/clipfrac_avg": 0.005786027293652296, + "policy/entropy_avg": 0.23768234252929688, + "step": 706, + "val/clipfrac_avg": 0.032500892877578735, + "val/num_eos_tokens": 203, + "val/ratio": 0.9860942959785461, + "val/ratio_var": 5.682241317117587e-05 + }, + { + "episode": 11312, + "epoch": 0.5713131313131313, + "eps": 0, + "loss/policy_avg": -0.5062837600708008, + "loss/value_avg": 0.7742470502853394, + "lr": 4.254788669119127e-06, + "objective/entropy": -93.2535400390625, + "objective/kl": 34.540130615234375, + "objective/non_score_reward": -1.727006435394287, + "objective/rlhf_reward": 14.788618087768555, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008475214708596468, + "policy/clipfrac_avg": 0.027025574818253517, + "policy/entropy_avg": 0.21597671508789062, + "step": 707, + "val/clipfrac_avg": 0.1775030940771103, + "val/num_eos_tokens": 165, + "val/ratio": 0.9894334077835083, + "val/ratio_var": 9.222733206115663e-05 + }, + { + "episode": 11328, + "epoch": 0.5721212121212121, + "eps": 0, + "loss/policy_avg": -0.1463669091463089, + "loss/value_avg": 0.5632686614990234, + "lr": 4.241583420204998e-06, + "objective/entropy": -110.37503051757812, + "objective/kl": 36.62909698486328, + "objective/non_score_reward": -1.831454873085022, + "objective/rlhf_reward": 12.89510726928711, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009251610608771443, + "policy/clipfrac_avg": 0.00941617414355278, + "policy/entropy_avg": 0.16801166534423828, + "step": 708, + "val/clipfrac_avg": 0.04514690116047859, + "val/num_eos_tokens": 267, + "val/ratio": 0.9897239208221436, + "val/ratio_var": 7.692311191931367e-05 + }, + { + "episode": 11344, + "epoch": 0.572929292929293, + "eps": 0, + "loss/policy_avg": -0.35295569896698, + "loss/value_avg": 0.7035583257675171, + "lr": 4.228383583725471e-06, + "objective/entropy": -76.27204132080078, + "objective/kl": 31.900005340576172, + "objective/non_score_reward": -1.5950002670288086, + "objective/rlhf_reward": 14.053436279296875, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010020419722422957, + "policy/clipfrac_avg": 0.011507621966302395, + "policy/entropy_avg": 0.16324424743652344, + "step": 709, + "val/clipfrac_avg": 0.12508229911327362, + "val/num_eos_tokens": 173, + "val/ratio": 0.9896982908248901, + "val/ratio_var": 4.9723090342013165e-05 + }, + { + "episode": 11360, + "epoch": 0.5737373737373738, + "eps": 0, + "loss/policy_avg": 0.160404771566391, + "loss/value_avg": 1.1914963722229004, + "lr": 4.215189253881093e-06, + "objective/entropy": -79.55085754394531, + "objective/kl": 36.81852722167969, + "objective/non_score_reward": -1.8409266471862793, + "objective/rlhf_reward": 12.159073829650879, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0011080782860517502, + "policy/clipfrac_avg": 0.006503967102617025, + "policy/entropy_avg": 0.19612884521484375, + "step": 710, + "val/clipfrac_avg": 0.08140315115451813, + "val/num_eos_tokens": 538, + "val/ratio": 0.9866147041320801, + "val/ratio_var": 7.869482942624018e-05 + }, + { + "episode": 11376, + "epoch": 0.5745454545454546, + "eps": 0, + "loss/policy_avg": 0.07548795640468597, + "loss/value_avg": 0.5138013362884521, + "lr": 4.2020005248331056e-06, + "objective/entropy": -119.03164672851562, + "objective/kl": 34.31198501586914, + "objective/non_score_reward": -1.7155992984771729, + "objective/rlhf_reward": 13.503150939941406, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007362160831689835, + "policy/clipfrac_avg": 0.00589254405349493, + "policy/entropy_avg": 0.14674091339111328, + "step": 711, + "val/clipfrac_avg": 0.02668866142630577, + "val/num_eos_tokens": 515, + "val/ratio": 0.990146279335022, + "val/ratio_var": 3.9673428545938805e-05 + }, + { + "episode": 11392, + "epoch": 0.5753535353535354, + "eps": 0, + "loss/policy_avg": -0.2925986051559448, + "loss/value_avg": 0.3821415603160858, + "lr": 4.1888174907027775e-06, + "objective/entropy": -112.06816101074219, + "objective/kl": 38.006500244140625, + "objective/non_score_reward": -1.900325059890747, + "objective/rlhf_reward": 13.990300178527832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.000874843040946871, + "policy/clipfrac_avg": 0.007544378750026226, + "policy/entropy_avg": 0.16875195503234863, + "step": 712, + "val/clipfrac_avg": 0.025153353810310364, + "val/num_eos_tokens": 4, + "val/ratio": 0.9893897771835327, + "val/ratio_var": 8.519255788996816e-05 + }, + { + "episode": 11408, + "epoch": 0.5761616161616162, + "eps": 0, + "loss/policy_avg": -0.5444605350494385, + "loss/value_avg": 1.517474889755249, + "lr": 4.175640245570742e-06, + "objective/entropy": -92.28114318847656, + "objective/kl": 35.69511413574219, + "objective/non_score_reward": -1.7847557067871094, + "objective/rlhf_reward": 13.73868179321289, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008441185345873237, + "policy/clipfrac_avg": 0.005670716054737568, + "policy/entropy_avg": 0.1591033935546875, + "step": 713, + "val/clipfrac_avg": 0.1791025698184967, + "val/num_eos_tokens": 98, + "val/ratio": 0.9903852939605713, + "val/ratio_var": 4.043114677187987e-05 + }, + { + "episode": 11424, + "epoch": 0.576969696969697, + "eps": 0, + "loss/policy_avg": -0.3059786260128021, + "loss/value_avg": 0.6795393228530884, + "lr": 4.162468883476319e-06, + "objective/entropy": -193.77468872070312, + "objective/kl": 42.17440414428711, + "objective/non_score_reward": -2.108720302581787, + "objective/rlhf_reward": 12.281905174255371, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008298080647364259, + "policy/clipfrac_avg": 0.00635810662060976, + "policy/entropy_avg": 0.15823698043823242, + "step": 714, + "val/clipfrac_avg": 0.0064142923802137375, + "val/num_eos_tokens": 1666, + "val/ratio": 0.9892975091934204, + "val/ratio_var": 6.228621350601315e-05 + }, + { + "episode": 11440, + "epoch": 0.5777777777777777, + "eps": 0, + "loss/policy_avg": -0.023257479071617126, + "loss/value_avg": 0.18199321627616882, + "lr": 4.149303498416838e-06, + "objective/entropy": -22.63348388671875, + "objective/kl": 44.308345794677734, + "objective/non_score_reward": -2.2154173851013184, + "objective/rlhf_reward": 14.82364559173584, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0012546201469376683, + "policy/clipfrac_avg": 0.007735411170870066, + "policy/entropy_avg": 0.240936279296875, + "step": 715, + "val/clipfrac_avg": 0.0028473599813878536, + "val/num_eos_tokens": 302, + "val/ratio": 0.9840776920318604, + "val/ratio_var": 5.949061960563995e-05 + }, + { + "episode": 11456, + "epoch": 0.5785858585858585, + "eps": 0, + "loss/policy_avg": 0.3237575888633728, + "loss/value_avg": 0.38197559118270874, + "lr": 4.136144184346979e-06, + "objective/entropy": -168.4746856689453, + "objective/kl": 40.63113784790039, + "objective/non_score_reward": -2.031557083129883, + "objective/rlhf_reward": 13.070005416870117, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0007934231543913484, + "policy/clipfrac_avg": 0.012050900608301163, + "policy/entropy_avg": 0.14413070678710938, + "step": 716, + "val/clipfrac_avg": 0.09322839975357056, + "val/num_eos_tokens": 1174, + "val/ratio": 0.9900047779083252, + "val/ratio_var": 5.29003118572291e-05 + }, + { + "episode": 11472, + "epoch": 0.5793939393939394, + "eps": 0, + "loss/policy_avg": 0.23654325306415558, + "loss/value_avg": 1.166879653930664, + "lr": 4.122991035178093e-06, + "objective/entropy": -96.77055358886719, + "objective/kl": 42.17526626586914, + "objective/non_score_reward": -2.1087632179260254, + "objective/rlhf_reward": 14.500611305236816, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000815618084743619, + "policy/clipfrac_avg": 0.008080050349235535, + "policy/entropy_avg": 0.14806270599365234, + "step": 717, + "val/clipfrac_avg": 0.02973807230591774, + "val/num_eos_tokens": 297, + "val/ratio": 0.9899397492408752, + "val/ratio_var": 3.871042645187117e-05 + }, + { + "episode": 11488, + "epoch": 0.5802020202020202, + "eps": 0, + "loss/policy_avg": -0.11968575417995453, + "loss/value_avg": 0.9426589608192444, + "lr": 4.109844144777537e-06, + "objective/entropy": -58.205047607421875, + "objective/kl": 42.6419792175293, + "objective/non_score_reward": -2.132099151611328, + "objective/rlhf_reward": 12.203838348388672, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009566791704855859, + "policy/clipfrac_avg": 0.0025268401950597763, + "policy/entropy_avg": 0.18893003463745117, + "step": 718, + "val/clipfrac_avg": 0.1098824143409729, + "val/num_eos_tokens": 16, + "val/ratio": 0.9869214296340942, + "val/ratio_var": 8.384165266761556e-05 + }, + { + "episode": 11504, + "epoch": 0.581010101010101, + "eps": 0, + "loss/policy_avg": 0.0208362378180027, + "loss/value_avg": 0.8499202728271484, + "lr": 4.096703606968007e-06, + "objective/entropy": -105.71292114257812, + "objective/kl": 40.70527648925781, + "objective/non_score_reward": -2.035263776779175, + "objective/rlhf_reward": 13.605360984802246, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008132618968375027, + "policy/clipfrac_avg": 0.004117591306567192, + "policy/entropy_avg": 0.15980243682861328, + "step": 719, + "val/clipfrac_avg": 0.04368842393159866, + "val/num_eos_tokens": 428, + "val/ratio": 0.9895676374435425, + "val/ratio_var": 6.342094275169075e-05 + }, + { + "episode": 11520, + "epoch": 0.5818181818181818, + "eps": 0, + "loss/policy_avg": -0.11627452820539474, + "loss/value_avg": 0.7368168830871582, + "lr": 4.083569515526853e-06, + "objective/entropy": -100.23733520507812, + "objective/kl": 32.93721008300781, + "objective/non_score_reward": -1.6468604803085327, + "objective/rlhf_reward": 15.134389877319336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008385346736758947, + "policy/clipfrac_avg": 0.009182492271065712, + "policy/entropy_avg": 0.17890548706054688, + "step": 720, + "val/clipfrac_avg": 0.06867726147174835, + "val/num_eos_tokens": 144, + "val/ratio": 0.9896779656410217, + "val/ratio_var": 7.192800694610924e-05 + }, + { + "episode": 11536, + "epoch": 0.5826262626262626, + "eps": 0, + "loss/policy_avg": -0.06581735610961914, + "loss/value_avg": 0.7933100461959839, + "lr": 4.070441964185428e-06, + "objective/entropy": -103.3985366821289, + "objective/kl": 39.985321044921875, + "objective/non_score_reward": -1.9992659091949463, + "objective/rlhf_reward": 14.547609329223633, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000881620217114687, + "policy/clipfrac_avg": 0.004334163852035999, + "policy/entropy_avg": 0.1855926513671875, + "step": 721, + "val/clipfrac_avg": 0.04114583134651184, + "val/num_eos_tokens": 161, + "val/ratio": 0.989449679851532, + "val/ratio_var": 6.215622852323577e-05 + }, + { + "episode": 11552, + "epoch": 0.5834343434343434, + "eps": 0, + "loss/policy_avg": 0.44041353464126587, + "loss/value_avg": 0.7198789119720459, + "lr": 4.05732104662841e-06, + "objective/entropy": -119.77040100097656, + "objective/kl": 36.42585754394531, + "objective/non_score_reward": -1.8212929964065552, + "objective/rlhf_reward": 13.467769622802734, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008070225012488663, + "policy/clipfrac_avg": 0.007022568956017494, + "policy/entropy_avg": 0.15318524837493896, + "step": 722, + "val/clipfrac_avg": 0.05163312703371048, + "val/num_eos_tokens": 304, + "val/ratio": 0.9899815320968628, + "val/ratio_var": 4.889031333732419e-05 + }, + { + "episode": 11568, + "epoch": 0.5842424242424242, + "eps": 0, + "loss/policy_avg": 0.4866091012954712, + "loss/value_avg": 1.1953067779541016, + "lr": 4.04420685649314e-06, + "objective/entropy": -50.46599578857422, + "objective/kl": 39.113037109375, + "objective/non_score_reward": -1.955651879310608, + "objective/rlhf_reward": 13.911535263061523, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010366745991632342, + "policy/clipfrac_avg": 0.006084335036575794, + "policy/entropy_avg": 0.20149445533752441, + "step": 723, + "val/clipfrac_avg": 0.09489050507545471, + "val/num_eos_tokens": 36, + "val/ratio": 0.9867548942565918, + "val/ratio_var": 6.938407750567421e-05 + }, + { + "episode": 11584, + "epoch": 0.585050505050505, + "eps": 0, + "loss/policy_avg": 0.1589544266462326, + "loss/value_avg": 0.568170964717865, + "lr": 4.031099487368937e-06, + "objective/entropy": -112.03081512451172, + "objective/kl": 37.355255126953125, + "objective/non_score_reward": -1.8677629232406616, + "objective/rlhf_reward": 13.49161148071289, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001035548048093915, + "policy/clipfrac_avg": 0.006865138188004494, + "policy/entropy_avg": 0.20564031600952148, + "step": 724, + "val/clipfrac_avg": 0.05233292281627655, + "val/num_eos_tokens": 347, + "val/ratio": 0.9859846830368042, + "val/ratio_var": 0.00010150864545721561 + }, + { + "episode": 11600, + "epoch": 0.5858585858585859, + "eps": 0, + "loss/policy_avg": -0.19635428488254547, + "loss/value_avg": 0.33023789525032043, + "lr": 4.0179990327964526e-06, + "objective/entropy": -69.79931640625, + "objective/kl": 41.90481948852539, + "objective/non_score_reward": -2.095241069793701, + "objective/rlhf_reward": 14.85007095336914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0010360007872805, + "policy/clipfrac_avg": 0.004560987465083599, + "policy/entropy_avg": 0.20290184020996094, + "step": 725, + "val/clipfrac_avg": 0.08375490456819534, + "val/num_eos_tokens": 261, + "val/ratio": 0.98649001121521, + "val/ratio_var": 6.110723916208372e-05 + }, + { + "episode": 11616, + "epoch": 0.5866666666666667, + "eps": 0, + "loss/policy_avg": 0.0999169647693634, + "loss/value_avg": 1.0625457763671875, + "lr": 4.004905586266988e-06, + "objective/entropy": -104.75285339355469, + "objective/kl": 32.01334762573242, + "objective/non_score_reward": -1.6006674766540527, + "objective/rlhf_reward": 12.766520500183105, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010034965816885233, + "policy/clipfrac_avg": 0.007322699762880802, + "policy/entropy_avg": 0.19988441467285156, + "step": 726, + "val/clipfrac_avg": 0.14562752842903137, + "val/num_eos_tokens": 33, + "val/ratio": 0.9887305498123169, + "val/ratio_var": 7.558859942946583e-05 + }, + { + "episode": 11632, + "epoch": 0.5874747474747475, + "eps": 0, + "loss/policy_avg": 0.004431502893567085, + "loss/value_avg": 0.5651569366455078, + "lr": 3.991819241221836e-06, + "objective/entropy": -53.65510940551758, + "objective/kl": 43.23419952392578, + "objective/non_score_reward": -2.161710262298584, + "objective/rlhf_reward": 12.986727714538574, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001010584644973278, + "policy/clipfrac_avg": 0.006680442951619625, + "policy/entropy_avg": 0.17978286743164062, + "step": 727, + "val/clipfrac_avg": 0.028638603165745735, + "val/num_eos_tokens": 25, + "val/ratio": 0.9880765676498413, + "val/ratio_var": 3.508182999212295e-05 + }, + { + "episode": 11648, + "epoch": 0.5882828282828283, + "eps": 0, + "loss/policy_avg": -0.026804931461811066, + "loss/value_avg": 0.3243325352668762, + "lr": 3.978740091051599e-06, + "objective/entropy": -147.60580444335938, + "objective/kl": 40.000404357910156, + "objective/non_score_reward": -2.0000202655792236, + "objective/rlhf_reward": 13.304666519165039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008220034651458263, + "policy/clipfrac_avg": 0.010559421963989735, + "policy/entropy_avg": 0.15192937850952148, + "step": 728, + "val/clipfrac_avg": 0.047058261930942535, + "val/num_eos_tokens": 93, + "val/ratio": 0.9890267252922058, + "val/ratio_var": 7.159873348427936e-05 + }, + { + "episode": 11664, + "epoch": 0.5890909090909091, + "eps": 0, + "loss/policy_avg": 0.061702877283096313, + "loss/value_avg": 0.25130608677864075, + "lr": 3.965668229095546e-06, + "objective/entropy": -75.26506042480469, + "objective/kl": 34.32489013671875, + "objective/non_score_reward": -1.7162445783615112, + "objective/rlhf_reward": 13.9556303024292, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008753975853323936, + "policy/clipfrac_avg": 0.006253514904528856, + "policy/entropy_avg": 0.17108154296875, + "step": 729, + "val/clipfrac_avg": 0.03564208000898361, + "val/num_eos_tokens": 194, + "val/ratio": 0.9883392453193665, + "val/ratio_var": 5.439678716356866e-05 + }, + { + "episode": 11680, + "epoch": 0.5898989898989899, + "eps": 0, + "loss/policy_avg": 0.5118429064750671, + "loss/value_avg": 0.3809865713119507, + "lr": 3.9526037486409275e-06, + "objective/entropy": -211.5462646484375, + "objective/kl": 38.28752136230469, + "objective/non_score_reward": -1.9143760204315186, + "objective/rlhf_reward": 13.062187194824219, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008200224256142974, + "policy/clipfrac_avg": 0.009272422641515732, + "policy/entropy_avg": 0.15618228912353516, + "step": 730, + "val/clipfrac_avg": 0.13058164715766907, + "val/num_eos_tokens": 191, + "val/ratio": 0.9892928600311279, + "val/ratio_var": 5.910802065045573e-05 + }, + { + "episode": 11696, + "epoch": 0.5907070707070707, + "eps": 0, + "loss/policy_avg": -0.575583279132843, + "loss/value_avg": 0.6588011384010315, + "lr": 3.939546742922318e-06, + "objective/entropy": -155.975341796875, + "objective/kl": 33.536251068115234, + "objective/non_score_reward": -1.6768126487731934, + "objective/rlhf_reward": 14.198186874389648, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006373925716616213, + "policy/clipfrac_avg": 0.011547036468982697, + "policy/entropy_avg": 0.15375041961669922, + "step": 731, + "val/clipfrac_avg": 0.1309666782617569, + "val/num_eos_tokens": 393, + "val/ratio": 0.9912899732589722, + "val/ratio_var": 7.494341116398573e-05 + }, + { + "episode": 11712, + "epoch": 0.5915151515151515, + "eps": 0, + "loss/policy_avg": 0.08975643664598465, + "loss/value_avg": 0.4319133758544922, + "lr": 3.926497305120943e-06, + "objective/entropy": -46.366912841796875, + "objective/kl": 38.771400451660156, + "objective/non_score_reward": -1.9385700225830078, + "objective/rlhf_reward": 14.631742477416992, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010276802349835634, + "policy/clipfrac_avg": 0.004419120494276285, + "policy/entropy_avg": 0.19952774047851562, + "step": 732, + "val/clipfrac_avg": 0.02601846680045128, + "val/num_eos_tokens": 121, + "val/ratio": 0.9856929779052734, + "val/ratio_var": 5.2173218136886135e-05 + }, + { + "episode": 11728, + "epoch": 0.5923232323232324, + "eps": 0, + "loss/policy_avg": -0.5671601295471191, + "loss/value_avg": 1.4009994268417358, + "lr": 3.913455528364027e-06, + "objective/entropy": -30.65240478515625, + "objective/kl": 34.644386291503906, + "objective/non_score_reward": -1.7322194576263428, + "objective/rlhf_reward": 16.127155303955078, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009229708230122924, + "policy/clipfrac_avg": 0.0053136469796299934, + "policy/entropy_avg": 0.1684565544128418, + "step": 733, + "val/clipfrac_avg": 0.16779303550720215, + "val/num_eos_tokens": 257, + "val/ratio": 0.98807692527771, + "val/ratio_var": 4.935808829031885e-05 + }, + { + "episode": 11744, + "epoch": 0.5931313131313132, + "eps": 0, + "loss/policy_avg": 0.1117314025759697, + "loss/value_avg": 0.6027213335037231, + "lr": 3.900421505724119e-06, + "objective/entropy": -115.830078125, + "objective/kl": 38.92997741699219, + "objective/non_score_reward": -1.9464988708496094, + "objective/rlhf_reward": 12.38162612915039, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006995117873884737, + "policy/clipfrac_avg": 0.0067988173104822636, + "policy/entropy_avg": 0.12786293029785156, + "step": 734, + "val/clipfrac_avg": 0.11902365833520889, + "val/num_eos_tokens": 256, + "val/ratio": 0.9913206696510315, + "val/ratio_var": 3.536290387273766e-05 + }, + { + "episode": 11760, + "epoch": 0.593939393939394, + "eps": 0, + "loss/policy_avg": 0.07707646489143372, + "loss/value_avg": 0.8557188510894775, + "lr": 3.887395330218429e-06, + "objective/entropy": -77.05795288085938, + "objective/kl": 34.79220199584961, + "objective/non_score_reward": -1.739610195159912, + "objective/rlhf_reward": 12.510390281677246, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009019911522045732, + "policy/clipfrac_avg": 0.00438590906560421, + "policy/entropy_avg": 0.18432998657226562, + "step": 735, + "val/clipfrac_avg": 0.20910227298736572, + "val/num_eos_tokens": 339, + "val/ratio": 0.9879709482192993, + "val/ratio_var": 7.31311592971906e-05 + }, + { + "episode": 11776, + "epoch": 0.5947474747474748, + "eps": 0, + "loss/policy_avg": 0.005743637681007385, + "loss/value_avg": 0.4823000729084015, + "lr": 3.8743770948081675e-06, + "objective/entropy": -189.58908081054688, + "objective/kl": 40.491024017333984, + "objective/non_score_reward": -2.0245513916015625, + "objective/rlhf_reward": 12.608261108398438, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010324950562790036, + "policy/clipfrac_avg": 0.008821255527436733, + "policy/entropy_avg": 0.1864786148071289, + "step": 736, + "val/clipfrac_avg": 0.03615584969520569, + "val/num_eos_tokens": 146, + "val/ratio": 0.9867680072784424, + "val/ratio_var": 0.00012203787628095597 + }, + { + "episode": 11792, + "epoch": 0.5955555555555555, + "eps": 0, + "loss/policy_avg": 0.11549818515777588, + "loss/value_avg": 0.4191131591796875, + "lr": 3.861366892397878e-06, + "objective/entropy": -53.57340621948242, + "objective/kl": 47.2681999206543, + "objective/non_score_reward": -2.363409996032715, + "objective/rlhf_reward": 11.238153457641602, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0010416920995339751, + "policy/clipfrac_avg": 0.006328020244836807, + "policy/entropy_avg": 0.21573257446289062, + "step": 737, + "val/clipfrac_avg": 0.05454040318727493, + "val/num_eos_tokens": 145, + "val/ratio": 0.9863517880439758, + "val/ratio_var": 5.17384396516718e-05 + }, + { + "episode": 11808, + "epoch": 0.5963636363636363, + "eps": 0, + "loss/policy_avg": 0.09252296388149261, + "loss/value_avg": 0.24368464946746826, + "lr": 3.848364815834782e-06, + "objective/entropy": -9.551445007324219, + "objective/kl": 41.81903839111328, + "objective/non_score_reward": -2.090951919555664, + "objective/rlhf_reward": 15.362173080444336, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.001077502267435193, + "policy/clipfrac_avg": 0.007244379259645939, + "policy/entropy_avg": 0.208465576171875, + "step": 738, + "val/clipfrac_avg": 0.046727634966373444, + "val/num_eos_tokens": 70, + "val/ratio": 0.9859708547592163, + "val/ratio_var": 4.752089807880111e-05 + }, + { + "episode": 11824, + "epoch": 0.5971717171717171, + "eps": 0, + "loss/policy_avg": -0.47144564986228943, + "loss/value_avg": 0.9659825563430786, + "lr": 3.835370957908108e-06, + "objective/entropy": -109.4366226196289, + "objective/kl": 32.577239990234375, + "objective/non_score_reward": -1.6288621425628662, + "objective/rlhf_reward": 13.332075119018555, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009311667527072132, + "policy/clipfrac_avg": 0.007746369112282991, + "policy/entropy_avg": 0.2024822235107422, + "step": 739, + "val/clipfrac_avg": 0.1542586386203766, + "val/num_eos_tokens": 140, + "val/ratio": 0.9888983964920044, + "val/ratio_var": 8.097994577838108e-05 + }, + { + "episode": 11840, + "epoch": 0.597979797979798, + "eps": 0, + "loss/policy_avg": 0.3934873938560486, + "loss/value_avg": 0.5160516500473022, + "lr": 3.822385411348429e-06, + "objective/entropy": -82.36312103271484, + "objective/kl": 38.616180419921875, + "objective/non_score_reward": -1.9308089017868042, + "objective/rlhf_reward": 12.725440979003906, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008991706417873502, + "policy/clipfrac_avg": 0.01843995228409767, + "policy/entropy_avg": 0.2245502471923828, + "step": 740, + "val/clipfrac_avg": 0.0067659346386790276, + "val/num_eos_tokens": 30, + "val/ratio": 0.988348662853241, + "val/ratio_var": 5.281575795379467e-05 + }, + { + "episode": 11856, + "epoch": 0.5987878787878788, + "eps": 0, + "loss/policy_avg": -0.06747608631849289, + "loss/value_avg": 0.3114325702190399, + "lr": 3.809408268827009e-06, + "objective/entropy": -89.22055053710938, + "objective/kl": 46.57951736450195, + "objective/non_score_reward": -2.3289759159088135, + "objective/rlhf_reward": 13.483524322509766, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000996086047962308, + "policy/clipfrac_avg": 0.005896539893001318, + "policy/entropy_avg": 0.19659423828125, + "step": 741, + "val/clipfrac_avg": 0.0699368417263031, + "val/num_eos_tokens": 107, + "val/ratio": 0.9875314235687256, + "val/ratio_var": 4.793483094545081e-05 + }, + { + "episode": 11872, + "epoch": 0.5995959595959596, + "eps": 0, + "loss/policy_avg": -0.2737995684146881, + "loss/value_avg": 0.4772384762763977, + "lr": 3.7964396229551365e-06, + "objective/entropy": -104.62603759765625, + "objective/kl": 36.626800537109375, + "objective/non_score_reward": -1.831339955329895, + "objective/rlhf_reward": 14.778034210205078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008760478813201189, + "policy/clipfrac_avg": 0.005249073728919029, + "policy/entropy_avg": 0.17531394958496094, + "step": 742, + "val/clipfrac_avg": 0.06328597664833069, + "val/num_eos_tokens": 91, + "val/ratio": 0.9885821342468262, + "val/ratio_var": 5.911157131777145e-05 + }, + { + "episode": 11888, + "epoch": 0.6004040404040404, + "eps": 0, + "loss/policy_avg": -0.11605305969715118, + "loss/value_avg": 0.30427810549736023, + "lr": 3.783479566283457e-06, + "objective/entropy": -123.75762176513672, + "objective/kl": 35.787086486816406, + "objective/non_score_reward": -1.7893544435501099, + "objective/rlhf_reward": 14.86689567565918, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007770034717395902, + "policy/clipfrac_avg": 0.0056176697835326195, + "policy/entropy_avg": 0.152313232421875, + "step": 743, + "val/clipfrac_avg": 0.048395052552223206, + "val/num_eos_tokens": 386, + "val/ratio": 0.9905653595924377, + "val/ratio_var": 5.709798642783426e-05 + }, + { + "episode": 11904, + "epoch": 0.6012121212121212, + "eps": 0, + "loss/policy_avg": -0.02294028550386429, + "loss/value_avg": 0.7253668904304504, + "lr": 3.7705281913013286e-06, + "objective/entropy": -39.3416633605957, + "objective/kl": 43.74962615966797, + "objective/non_score_reward": -2.18748140335083, + "objective/rlhf_reward": 12.648456573486328, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010895703453570604, + "policy/clipfrac_avg": 0.0052008070051670074, + "policy/entropy_avg": 0.22774124145507812, + "step": 744, + "val/clipfrac_avg": 0.01829145848751068, + "val/num_eos_tokens": 71, + "val/ratio": 0.985338568687439, + "val/ratio_var": 6.19284255662933e-05 + }, + { + "episode": 11920, + "epoch": 0.602020202020202, + "eps": 0, + "loss/policy_avg": 0.1328388750553131, + "loss/value_avg": 1.3390746116638184, + "lr": 3.7575855904361447e-06, + "objective/entropy": -79.38550567626953, + "objective/kl": 33.780616760253906, + "objective/non_score_reward": -1.6890308856964111, + "objective/rlhf_reward": 14.178156852722168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008661438478156924, + "policy/clipfrac_avg": 0.003558866214007139, + "policy/entropy_avg": 0.15964603424072266, + "step": 745, + "val/clipfrac_avg": 0.05471283197402954, + "val/num_eos_tokens": 81, + "val/ratio": 0.9897934198379517, + "val/ratio_var": 8.215933485189453e-05 + }, + { + "episode": 11936, + "epoch": 0.6028282828282828, + "eps": 0, + "loss/policy_avg": -0.29737263917922974, + "loss/value_avg": 0.6314270496368408, + "lr": 3.7446518560526903e-06, + "objective/entropy": -124.25365447998047, + "objective/kl": 42.908660888671875, + "objective/non_score_reward": -2.145432949066162, + "objective/rlhf_reward": 11.87019157409668, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009470224613323808, + "policy/clipfrac_avg": 0.006878838874399662, + "policy/entropy_avg": 0.1818103790283203, + "step": 746, + "val/clipfrac_avg": 0.0798804834485054, + "val/num_eos_tokens": 329, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 8.261724724434316e-05 + }, + { + "episode": 11952, + "epoch": 0.6036363636363636, + "eps": 0, + "loss/policy_avg": 0.12454540282487869, + "loss/value_avg": 0.4487236738204956, + "lr": 3.731727080452464e-06, + "objective/entropy": -72.31340026855469, + "objective/kl": 43.510276794433594, + "objective/non_score_reward": -2.175513982772827, + "objective/rlhf_reward": 12.347923278808594, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009941391181200743, + "policy/clipfrac_avg": 0.00658874586224556, + "policy/entropy_avg": 0.16569900512695312, + "step": 747, + "val/clipfrac_avg": 0.1540203094482422, + "val/num_eos_tokens": 77, + "val/ratio": 0.9888790845870972, + "val/ratio_var": 2.4041321012191474e-05 + }, + { + "episode": 11968, + "epoch": 0.6044444444444445, + "eps": 0, + "loss/policy_avg": -0.1547861397266388, + "loss/value_avg": 0.49022629857063293, + "lr": 3.7188113558730403e-06, + "objective/entropy": -103.35200500488281, + "objective/kl": 35.28123474121094, + "objective/non_score_reward": -1.764061689376831, + "objective/rlhf_reward": 13.27500057220459, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008191847009584308, + "policy/clipfrac_avg": 0.006343110464513302, + "policy/entropy_avg": 0.14904117584228516, + "step": 748, + "val/clipfrac_avg": 0.04430587589740753, + "val/num_eos_tokens": 124, + "val/ratio": 0.9908789396286011, + "val/ratio_var": 4.545380579656921e-05 + }, + { + "episode": 11984, + "epoch": 0.6052525252525253, + "eps": 0, + "loss/policy_avg": 0.11601711809635162, + "loss/value_avg": 0.6175556182861328, + "lr": 3.705904774487396e-06, + "objective/entropy": -111.66493225097656, + "objective/kl": 36.76349639892578, + "objective/non_score_reward": -1.838174819946289, + "objective/rlhf_reward": 13.693075180053711, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008918631356209517, + "policy/clipfrac_avg": 0.010910662822425365, + "policy/entropy_avg": 0.16454505920410156, + "step": 749, + "val/clipfrac_avg": 0.06755826622247696, + "val/num_eos_tokens": 358, + "val/ratio": 0.9886500835418701, + "val/ratio_var": 6.255816697375849e-05 + }, + { + "episode": 12000, + "epoch": 0.6060606060606061, + "eps": 0, + "loss/policy_avg": -0.04627024754881859, + "loss/value_avg": 0.6224689483642578, + "lr": 3.6930074284032613e-06, + "objective/entropy": -63.2913703918457, + "objective/kl": 32.27269744873047, + "objective/non_score_reward": -1.6136349439620972, + "objective/rlhf_reward": 14.644177436828613, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009490256779827178, + "policy/clipfrac_avg": 0.0068245502188801765, + "policy/entropy_avg": 0.17229938507080078, + "step": 750, + "val/clipfrac_avg": 0.18630267679691315, + "val/num_eos_tokens": 11, + "val/ratio": 0.9878020882606506, + "val/ratio_var": 5.455148493638262e-05 + }, + { + "episode": 12016, + "epoch": 0.6068686868686869, + "eps": 0, + "loss/policy_avg": -0.17890788614749908, + "loss/value_avg": 0.7140947580337524, + "lr": 3.6801194096624515e-06, + "objective/entropy": -54.77946472167969, + "objective/kl": 34.44614028930664, + "objective/non_score_reward": -1.7223069667816162, + "objective/rlhf_reward": 14.894880294799805, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011779924388974905, + "policy/clipfrac_avg": 0.024402525275945663, + "policy/entropy_avg": 0.2098255157470703, + "step": 751, + "val/clipfrac_avg": 0.04182715713977814, + "val/num_eos_tokens": 42, + "val/ratio": 0.9854939579963684, + "val/ratio_var": 0.00014133901277091354 + }, + { + "episode": 12032, + "epoch": 0.6076767676767677, + "eps": 0, + "loss/policy_avg": -0.0015007667243480682, + "loss/value_avg": 0.2544102668762207, + "lr": 3.6672408102402247e-06, + "objective/entropy": -70.776611328125, + "objective/kl": 37.67994689941406, + "objective/non_score_reward": -1.8839974403381348, + "objective/rlhf_reward": 12.108190536499023, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009373503271490335, + "policy/clipfrac_avg": 0.005210112780332565, + "policy/entropy_avg": 0.18710041046142578, + "step": 752, + "val/clipfrac_avg": 0.021128622815012932, + "val/num_eos_tokens": 218, + "val/ratio": 0.9874061942100525, + "val/ratio_var": 5.212551332078874e-05 + }, + { + "episode": 12048, + "epoch": 0.6084848484848485, + "eps": 0, + "loss/policy_avg": 0.05351484939455986, + "loss/value_avg": 0.28849858045578003, + "lr": 3.654371722044616e-06, + "objective/entropy": -89.17417907714844, + "objective/kl": 44.5589714050293, + "objective/non_score_reward": -2.2279486656188965, + "objective/rlhf_reward": 12.178300857543945, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009375615045428276, + "policy/clipfrac_avg": 0.005882620811462402, + "policy/entropy_avg": 0.18878555297851562, + "step": 753, + "val/clipfrac_avg": 0.021566759794950485, + "val/num_eos_tokens": 120, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 5.727734605898149e-05 + }, + { + "episode": 12064, + "epoch": 0.6092929292929293, + "eps": 0, + "loss/policy_avg": -0.12424173951148987, + "loss/value_avg": 0.359388530254364, + "lr": 3.641512236915783e-06, + "objective/entropy": -114.3979263305664, + "objective/kl": 43.28697967529297, + "objective/non_score_reward": -2.16434907913208, + "objective/rlhf_reward": 14.226276397705078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000866133370436728, + "policy/clipfrac_avg": 0.0050315093249082565, + "policy/entropy_avg": 0.16837501525878906, + "step": 754, + "val/clipfrac_avg": 0.1426960825920105, + "val/num_eos_tokens": 391, + "val/ratio": 0.9894628524780273, + "val/ratio_var": 5.016153590986505e-05 + }, + { + "episode": 12080, + "epoch": 0.6101010101010101, + "eps": 0, + "loss/policy_avg": -0.2821425795555115, + "loss/value_avg": 0.38819342851638794, + "lr": 3.6286624466253496e-06, + "objective/entropy": -164.57290649414062, + "objective/kl": 38.83340072631836, + "objective/non_score_reward": -1.9416700601577759, + "objective/rlhf_reward": 14.245829582214355, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008140793070197105, + "policy/clipfrac_avg": 0.00435154652222991, + "policy/entropy_avg": 0.15729975700378418, + "step": 755, + "val/clipfrac_avg": 0.09525477886199951, + "val/num_eos_tokens": 336, + "val/ratio": 0.989446222782135, + "val/ratio_var": 5.3185020078672096e-05 + }, + { + "episode": 12096, + "epoch": 0.610909090909091, + "eps": 0, + "loss/policy_avg": 0.05094696208834648, + "loss/value_avg": 0.4260476529598236, + "lr": 3.6158224428757538e-06, + "objective/entropy": -61.176116943359375, + "objective/kl": 32.80990219116211, + "objective/non_score_reward": -1.6404950618743896, + "objective/rlhf_reward": 16.24231719970703, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009597806492820382, + "policy/clipfrac_avg": 0.0061972904950380325, + "policy/entropy_avg": 0.16566753387451172, + "step": 756, + "val/clipfrac_avg": 0.01815173029899597, + "val/num_eos_tokens": 163, + "val/ratio": 0.9894044399261475, + "val/ratio_var": 4.711053770733997e-05 + }, + { + "episode": 12112, + "epoch": 0.6117171717171718, + "eps": 0, + "loss/policy_avg": -0.040469832718372345, + "loss/value_avg": 0.3746616244316101, + "lr": 3.602992317299594e-06, + "objective/entropy": -52.30070877075195, + "objective/kl": 34.218990325927734, + "objective/non_score_reward": -1.7109496593475342, + "objective/rlhf_reward": 13.273425102233887, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009450222714804113, + "policy/clipfrac_avg": 0.0049097794108092785, + "policy/entropy_avg": 0.1805877685546875, + "step": 757, + "val/clipfrac_avg": 0.09231511503458023, + "val/num_eos_tokens": 105, + "val/ratio": 0.9877369403839111, + "val/ratio_var": 4.407649248605594e-05 + }, + { + "episode": 12128, + "epoch": 0.6125252525252525, + "eps": 0, + "loss/policy_avg": -0.0790981650352478, + "loss/value_avg": 0.289002388715744, + "lr": 3.5901721614589725e-06, + "objective/entropy": -33.08611297607422, + "objective/kl": 42.89472961425781, + "objective/non_score_reward": -2.1447365283966064, + "objective/rlhf_reward": 14.792763710021973, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011628612410277128, + "policy/clipfrac_avg": 0.00601824838668108, + "policy/entropy_avg": 0.21850204467773438, + "step": 758, + "val/clipfrac_avg": 0.05997534096240997, + "val/num_eos_tokens": 208, + "val/ratio": 0.9858998656272888, + "val/ratio_var": 4.6025765186641365e-05 + }, + { + "episode": 12144, + "epoch": 0.6133333333333333, + "eps": 0, + "loss/policy_avg": 0.08223091810941696, + "loss/value_avg": 0.4521709978580475, + "lr": 3.5773620668448384e-06, + "objective/entropy": -57.002418518066406, + "objective/kl": 34.86107635498047, + "objective/non_score_reward": -1.743053674697876, + "objective/rlhf_reward": 12.421009063720703, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009696290362626314, + "policy/clipfrac_avg": 0.005794525146484375, + "policy/entropy_avg": 0.1826324462890625, + "step": 759, + "val/clipfrac_avg": 0.03934604302048683, + "val/num_eos_tokens": 137, + "val/ratio": 0.9880379438400269, + "val/ratio_var": 4.820780668524094e-05 + }, + { + "episode": 12160, + "epoch": 0.6141414141414141, + "eps": 0, + "loss/policy_avg": 0.012903919443488121, + "loss/value_avg": 0.22128555178642273, + "lr": 3.564562124876344e-06, + "objective/entropy": -58.77183532714844, + "objective/kl": 40.15751647949219, + "objective/non_score_reward": -2.007875919342041, + "objective/rlhf_reward": 13.171812057495117, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009755170904099941, + "policy/clipfrac_avg": 0.0075699929147958755, + "policy/entropy_avg": 0.1793661117553711, + "step": 760, + "val/clipfrac_avg": 0.007695037871599197, + "val/num_eos_tokens": 194, + "val/ratio": 0.9879107475280762, + "val/ratio_var": 9.122795017901808e-05 + }, + { + "episode": 12176, + "epoch": 0.6149494949494949, + "eps": 0, + "loss/policy_avg": 0.23269692063331604, + "loss/value_avg": 0.6034168004989624, + "lr": 3.5517724269001853e-06, + "objective/entropy": -92.11294555664062, + "objective/kl": 36.40907287597656, + "objective/non_score_reward": -1.8204538822174072, + "objective/rlhf_reward": 13.468608856201172, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009949133964255452, + "policy/clipfrac_avg": 0.012031201273202896, + "policy/entropy_avg": 0.19497203826904297, + "step": 761, + "val/clipfrac_avg": 0.03142271935939789, + "val/num_eos_tokens": 210, + "val/ratio": 0.9871051907539368, + "val/ratio_var": 7.474326412193477e-05 + }, + { + "episode": 12192, + "epoch": 0.6157575757575757, + "eps": 0, + "loss/policy_avg": -0.014487015083432198, + "loss/value_avg": 1.0809375047683716, + "lr": 3.538993064189954e-06, + "objective/entropy": -42.83490753173828, + "objective/kl": 32.94031524658203, + "objective/non_score_reward": -1.6470158100128174, + "objective/rlhf_reward": 13.567828178405762, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0011591427028179169, + "policy/clipfrac_avg": 0.004484353121370077, + "policy/entropy_avg": 0.21469688415527344, + "step": 762, + "val/clipfrac_avg": 0.1017560362815857, + "val/num_eos_tokens": 453, + "val/ratio": 0.9864501953125, + "val/ratio_var": 8.947903552325442e-05 + }, + { + "episode": 12208, + "epoch": 0.6165656565656565, + "eps": 0, + "loss/policy_avg": -0.0410991832613945, + "loss/value_avg": 1.3014378547668457, + "lr": 3.526224127945479e-06, + "objective/entropy": -71.51974487304688, + "objective/kl": 31.28769302368164, + "objective/non_score_reward": -1.5643846988677979, + "objective/rlhf_reward": 15.216865539550781, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007811100222170353, + "policy/clipfrac_avg": 0.0053521376103162766, + "policy/entropy_avg": 0.16764509677886963, + "step": 763, + "val/clipfrac_avg": 0.059704042971134186, + "val/num_eos_tokens": 203, + "val/ratio": 0.9894566535949707, + "val/ratio_var": 6.841208232799545e-05 + }, + { + "episode": 12224, + "epoch": 0.6173737373737374, + "eps": 0, + "loss/policy_avg": 0.112126424908638, + "loss/value_avg": 0.725554347038269, + "lr": 3.5134657092921877e-06, + "objective/entropy": -99.9854736328125, + "objective/kl": 41.09782028198242, + "objective/non_score_reward": -2.0548908710479736, + "objective/rlhf_reward": 10.624795913696289, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009035035036504269, + "policy/clipfrac_avg": 0.010427422821521759, + "policy/entropy_avg": 0.17998504638671875, + "step": 764, + "val/clipfrac_avg": 0.12764863669872284, + "val/num_eos_tokens": 226, + "val/ratio": 0.988477349281311, + "val/ratio_var": 8.909796451916918e-05 + }, + { + "episode": 12240, + "epoch": 0.6181818181818182, + "eps": 0, + "loss/policy_avg": 0.37758830189704895, + "loss/value_avg": 0.7465800046920776, + "lr": 3.500717899280442e-06, + "objective/entropy": -135.23390197753906, + "objective/kl": 41.466583251953125, + "objective/non_score_reward": -2.073329210281372, + "objective/rlhf_reward": 12.94229507446289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009621024364605546, + "policy/clipfrac_avg": 0.00628144945949316, + "policy/entropy_avg": 0.17942047119140625, + "step": 765, + "val/clipfrac_avg": 0.10096652805805206, + "val/num_eos_tokens": 233, + "val/ratio": 0.9877568483352661, + "val/ratio_var": 7.067532715154812e-05 + }, + { + "episode": 12256, + "epoch": 0.618989898989899, + "eps": 0, + "loss/policy_avg": -0.10782650113105774, + "loss/value_avg": 0.4996359944343567, + "lr": 3.4879807888849026e-06, + "objective/entropy": -86.19502258300781, + "objective/kl": 39.61420440673828, + "objective/non_score_reward": -1.9807100296020508, + "objective/rlhf_reward": 13.706789016723633, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009516358841210604, + "policy/clipfrac_avg": 0.01008635200560093, + "policy/entropy_avg": 0.17633438110351562, + "step": 766, + "val/clipfrac_avg": 0.05575938522815704, + "val/num_eos_tokens": 426, + "val/ratio": 0.9884860515594482, + "val/ratio_var": 5.413329563452862e-05 + }, + { + "episode": 12272, + "epoch": 0.6197979797979798, + "eps": 0, + "loss/policy_avg": 0.0611935630440712, + "loss/value_avg": 0.40072089433670044, + "lr": 3.475254469003865e-06, + "objective/entropy": -78.26728057861328, + "objective/kl": 38.48719787597656, + "objective/non_score_reward": -1.9243600368499756, + "objective/rlhf_reward": 11.661577224731445, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009466722258366644, + "policy/clipfrac_avg": 0.0042708818800747395, + "policy/entropy_avg": 0.17498779296875, + "step": 767, + "val/clipfrac_avg": 0.1151452288031578, + "val/num_eos_tokens": 415, + "val/ratio": 0.9874659776687622, + "val/ratio_var": 7.443072536261752e-05 + }, + { + "episode": 12288, + "epoch": 0.6206060606060606, + "eps": 0, + "loss/policy_avg": -0.07818017899990082, + "loss/value_avg": 0.588293731212616, + "lr": 3.4625390304586224e-06, + "objective/entropy": -45.31693649291992, + "objective/kl": 41.06260681152344, + "objective/non_score_reward": -2.0531303882598877, + "objective/rlhf_reward": 13.860933303833008, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0011651598615571856, + "policy/clipfrac_avg": 0.0047182347625494, + "policy/entropy_avg": 0.2090606689453125, + "step": 768, + "val/clipfrac_avg": 0.16338574886322021, + "val/num_eos_tokens": 9, + "val/ratio": 0.986497700214386, + "val/ratio_var": 8.175932453013957e-05 + }, + { + "episode": 12304, + "epoch": 0.6214141414141414, + "eps": 0, + "loss/policy_avg": -0.3265213966369629, + "loss/value_avg": 0.5878744125366211, + "lr": 3.4498345639928167e-06, + "objective/entropy": -103.96510314941406, + "objective/kl": 35.794551849365234, + "objective/non_score_reward": -1.7897276878356934, + "objective/rlhf_reward": 13.507146835327148, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010084104724228382, + "policy/clipfrac_avg": 0.0070042358711361885, + "policy/entropy_avg": 0.19075870513916016, + "step": 769, + "val/clipfrac_avg": 0.12878292798995972, + "val/num_eos_tokens": 8, + "val/ratio": 0.9861502051353455, + "val/ratio_var": 8.973718649940565e-05 + }, + { + "episode": 12320, + "epoch": 0.6222222222222222, + "eps": 0, + "loss/policy_avg": -0.24864813685417175, + "loss/value_avg": 0.703667402267456, + "lr": 3.4371411602717785e-06, + "objective/entropy": -92.59934997558594, + "objective/kl": 31.87890625, + "objective/non_score_reward": -1.5939452648162842, + "objective/rlhf_reward": 14.429492950439453, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007836055592633784, + "policy/clipfrac_avg": 0.004061062820255756, + "policy/entropy_avg": 0.13613224029541016, + "step": 770, + "val/clipfrac_avg": 0.14356926083564758, + "val/num_eos_tokens": 22, + "val/ratio": 0.9909623265266418, + "val/ratio_var": 6.252773891901597e-05 + }, + { + "episode": 12336, + "epoch": 0.623030303030303, + "eps": 0, + "loss/policy_avg": -0.11106158792972565, + "loss/value_avg": 0.7719787359237671, + "lr": 3.424458909881897e-06, + "objective/entropy": -201.825927734375, + "objective/kl": 32.05082321166992, + "objective/non_score_reward": -1.602541208267212, + "objective/rlhf_reward": 13.842771530151367, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009224842651747167, + "policy/clipfrac_avg": 0.004090993665158749, + "policy/entropy_avg": 0.19452500343322754, + "step": 771, + "val/clipfrac_avg": 0.06381858140230179, + "val/num_eos_tokens": 137, + "val/ratio": 0.9873221516609192, + "val/ratio_var": 0.00010800129530252889 + }, + { + "episode": 12352, + "epoch": 0.6238383838383839, + "eps": 0, + "loss/policy_avg": -0.3147702217102051, + "loss/value_avg": 1.0198739767074585, + "lr": 3.411787903329963e-06, + "objective/entropy": -97.80130767822266, + "objective/kl": 28.94087791442871, + "objective/non_score_reward": -1.4470438957214355, + "objective/rlhf_reward": 13.572486877441406, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008233180269598961, + "policy/clipfrac_avg": 0.007523335050791502, + "policy/entropy_avg": 0.17452716827392578, + "step": 772, + "val/clipfrac_avg": 0.021688329055905342, + "val/num_eos_tokens": 116, + "val/ratio": 0.9885031580924988, + "val/ratio_var": 6.793635111534968e-05 + }, + { + "episode": 12368, + "epoch": 0.6246464646464647, + "eps": 0, + "loss/policy_avg": -0.019919998943805695, + "loss/value_avg": 0.31691867113113403, + "lr": 3.3991282310425276e-06, + "objective/entropy": -61.794166564941406, + "objective/kl": 33.001129150390625, + "objective/non_score_reward": -1.6500564813613892, + "objective/rlhf_reward": 12.943693161010742, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009417567634955049, + "policy/clipfrac_avg": 0.005282199941575527, + "policy/entropy_avg": 0.1732311248779297, + "step": 773, + "val/clipfrac_avg": 0.05137361213564873, + "val/num_eos_tokens": 13, + "val/ratio": 0.9889352321624756, + "val/ratio_var": 3.9570189983351156e-05 + }, + { + "episode": 12384, + "epoch": 0.6254545454545455, + "eps": 0, + "loss/policy_avg": 0.010051839053630829, + "loss/value_avg": 0.5414019227027893, + "lr": 3.386479983365251e-06, + "objective/entropy": -73.91504669189453, + "objective/kl": 29.43255615234375, + "objective/non_score_reward": -1.4716278314590454, + "objective/rlhf_reward": 13.293996810913086, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0010180522222071886, + "policy/clipfrac_avg": 0.0050889430567622185, + "policy/entropy_avg": 0.2153911590576172, + "step": 774, + "val/clipfrac_avg": 0.041913747787475586, + "val/num_eos_tokens": 214, + "val/ratio": 0.986315906047821, + "val/ratio_var": 8.102346328087151e-05 + }, + { + "episode": 12400, + "epoch": 0.6262626262626263, + "eps": 0, + "loss/policy_avg": -0.040474407374858856, + "loss/value_avg": 0.30427631735801697, + "lr": 3.3738432505622653e-06, + "objective/entropy": -75.78406524658203, + "objective/kl": 31.846160888671875, + "objective/non_score_reward": -1.5923081636428833, + "objective/rlhf_reward": 11.946754455566406, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009044273756444454, + "policy/clipfrac_avg": 0.005002772435545921, + "policy/entropy_avg": 0.18775272369384766, + "step": 775, + "val/clipfrac_avg": 0.04153668135404587, + "val/num_eos_tokens": 243, + "val/ratio": 0.9877529144287109, + "val/ratio_var": 5.462392800836824e-05 + }, + { + "episode": 12416, + "epoch": 0.6270707070707071, + "eps": 0, + "loss/policy_avg": -0.25257226824760437, + "loss/value_avg": 0.47636741399765015, + "lr": 3.3612181228155237e-06, + "objective/entropy": -97.84140014648438, + "objective/kl": 35.974266052246094, + "objective/non_score_reward": -1.798713207244873, + "objective/rlhf_reward": 13.685661315917969, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009363207500427961, + "policy/clipfrac_avg": 0.0035052683670073748, + "policy/entropy_avg": 0.18935298919677734, + "step": 776, + "val/clipfrac_avg": 0.07260937243700027, + "val/num_eos_tokens": 26, + "val/ratio": 0.9877586364746094, + "val/ratio_var": 0.00010292992374161258 + }, + { + "episode": 12432, + "epoch": 0.6278787878787879, + "eps": 0, + "loss/policy_avg": 0.41764986515045166, + "loss/value_avg": 0.8629061579704285, + "lr": 3.3486046902241663e-06, + "objective/entropy": -40.24072265625, + "objective/kl": 29.197486877441406, + "objective/non_score_reward": -1.4598743915557861, + "objective/rlhf_reward": 12.883874893188477, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010831760009750724, + "policy/clipfrac_avg": 0.02700810506939888, + "policy/entropy_avg": 0.16212844848632812, + "step": 777, + "val/clipfrac_avg": 0.08423963189125061, + "val/num_eos_tokens": 192, + "val/ratio": 0.9885735511779785, + "val/ratio_var": 8.724266081117094e-05 + }, + { + "episode": 12448, + "epoch": 0.6286868686868687, + "eps": 0, + "loss/policy_avg": 0.2242026925086975, + "loss/value_avg": 0.9421164989471436, + "lr": 3.336003042803861e-06, + "objective/entropy": -69.478271484375, + "objective/kl": 33.57006072998047, + "objective/non_score_reward": -1.6785030364990234, + "objective/rlhf_reward": 14.055871963500977, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008899248205125332, + "policy/clipfrac_avg": 0.006825867109000683, + "policy/entropy_avg": 0.14767837524414062, + "step": 778, + "val/clipfrac_avg": 0.1501787304878235, + "val/num_eos_tokens": 109, + "val/ratio": 0.9905418753623962, + "val/ratio_var": 4.142183388466947e-05 + }, + { + "episode": 12464, + "epoch": 0.6294949494949494, + "eps": 0, + "loss/policy_avg": 0.03614666312932968, + "loss/value_avg": 0.7387021780014038, + "lr": 3.3234132704861786e-06, + "objective/entropy": -182.4083251953125, + "objective/kl": 31.944690704345703, + "objective/non_score_reward": -1.5972344875335693, + "objective/rlhf_reward": 12.043390274047852, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.000871892087161541, + "policy/clipfrac_avg": 0.004613575059920549, + "policy/entropy_avg": 0.1733226776123047, + "step": 779, + "val/clipfrac_avg": 0.1070820540189743, + "val/num_eos_tokens": 28, + "val/ratio": 0.9880519509315491, + "val/ratio_var": 9.137512097368017e-05 + }, + { + "episode": 12480, + "epoch": 0.6303030303030303, + "eps": 0, + "loss/policy_avg": 0.491243839263916, + "loss/value_avg": 1.7449349164962769, + "lr": 3.310835463117942e-06, + "objective/entropy": -80.43102264404297, + "objective/kl": 22.97904396057129, + "objective/non_score_reward": -1.1489522457122803, + "objective/rlhf_reward": 13.28073501586914, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000753985601477325, + "policy/clipfrac_avg": 0.013304230757057667, + "policy/entropy_avg": 0.16235482692718506, + "step": 780, + "val/clipfrac_avg": 0.12828537821769714, + "val/num_eos_tokens": 261, + "val/ratio": 0.991666316986084, + "val/ratio_var": 4.57862151961308e-05 + }, + { + "episode": 12496, + "epoch": 0.6311111111111111, + "eps": 0, + "loss/policy_avg": -0.02106083557009697, + "loss/value_avg": 0.26465463638305664, + "lr": 3.2982697104605855e-06, + "objective/entropy": -96.58609008789062, + "objective/kl": 37.96983337402344, + "objective/non_score_reward": -1.898491621017456, + "objective/rlhf_reward": 12.742133140563965, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008919077808968723, + "policy/clipfrac_avg": 0.004138793330639601, + "policy/entropy_avg": 0.176055908203125, + "step": 781, + "val/clipfrac_avg": 0.022776879370212555, + "val/num_eos_tokens": 232, + "val/ratio": 0.9880313873291016, + "val/ratio_var": 5.30094766872935e-05 + }, + { + "episode": 12512, + "epoch": 0.6319191919191919, + "eps": 0, + "loss/policy_avg": -0.21595685184001923, + "loss/value_avg": 0.1668311059474945, + "lr": 3.2857161021895125e-06, + "objective/entropy": -113.97618103027344, + "objective/kl": 30.898021697998047, + "objective/non_score_reward": -1.5449011325836182, + "objective/rlhf_reward": 15.892599105834961, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0008122409926727414, + "policy/clipfrac_avg": 0.007180417887866497, + "policy/entropy_avg": 0.15228557586669922, + "step": 782, + "val/clipfrac_avg": 0.04710119590163231, + "val/num_eos_tokens": 88, + "val/ratio": 0.9905120134353638, + "val/ratio_var": 4.6862616727594286e-05 + }, + { + "episode": 12528, + "epoch": 0.6327272727272727, + "eps": 0, + "loss/policy_avg": -0.02830284647643566, + "loss/value_avg": 0.470698744058609, + "lr": 3.273174727893463e-06, + "objective/entropy": -119.84375762939453, + "objective/kl": 35.46149444580078, + "objective/non_score_reward": -1.773074746131897, + "objective/rlhf_reward": 13.00036334991455, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000846280949190259, + "policy/clipfrac_avg": 0.006307992152869701, + "policy/entropy_avg": 0.15985870361328125, + "step": 783, + "val/clipfrac_avg": 0.09303553402423859, + "val/num_eos_tokens": 242, + "val/ratio": 0.9886760711669922, + "val/ratio_var": 8.341170905623585e-05 + }, + { + "episode": 12544, + "epoch": 0.6335353535353535, + "eps": 0, + "loss/policy_avg": -0.03483296558260918, + "loss/value_avg": 0.2278536707162857, + "lr": 3.2606456770738636e-06, + "objective/entropy": -32.83402633666992, + "objective/kl": 35.053001403808594, + "objective/non_score_reward": -1.7526500225067139, + "objective/rlhf_reward": 13.708287239074707, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010523913661018014, + "policy/clipfrac_avg": 0.005916148889809847, + "policy/entropy_avg": 0.2171039581298828, + "step": 784, + "val/clipfrac_avg": 0.007893375121057034, + "val/num_eos_tokens": 463, + "val/ratio": 0.9859672784805298, + "val/ratio_var": 5.139111090102233e-05 + }, + { + "episode": 12560, + "epoch": 0.6343434343434343, + "eps": 0, + "loss/policy_avg": 0.17171788215637207, + "loss/value_avg": 0.5937256217002869, + "lr": 3.2481290391442034e-06, + "objective/entropy": -109.50154113769531, + "objective/kl": 41.38114929199219, + "objective/non_score_reward": -2.0690574645996094, + "objective/rlhf_reward": 13.82156753540039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009320294484496117, + "policy/clipfrac_avg": 0.005337849725037813, + "policy/entropy_avg": 0.1828467845916748, + "step": 785, + "val/clipfrac_avg": 0.0434221550822258, + "val/num_eos_tokens": 252, + "val/ratio": 0.9880971908569336, + "val/ratio_var": 7.803420157870278e-05 + }, + { + "episode": 12576, + "epoch": 0.6351515151515151, + "eps": 0, + "loss/policy_avg": -0.12326152622699738, + "loss/value_avg": 0.22453081607818604, + "lr": 3.235624903429374e-06, + "objective/entropy": -80.62982940673828, + "objective/kl": 39.33965301513672, + "objective/non_score_reward": -1.9669828414916992, + "objective/rlhf_reward": 14.3767671585083, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009726607240736485, + "policy/clipfrac_avg": 0.005861825309693813, + "policy/entropy_avg": 0.18674850463867188, + "step": 786, + "val/clipfrac_avg": 0.04197417199611664, + "val/num_eos_tokens": 55, + "val/ratio": 0.9867269396781921, + "val/ratio_var": 3.261497477069497e-05 + }, + { + "episode": 12592, + "epoch": 0.635959595959596, + "eps": 0, + "loss/policy_avg": 0.02891068160533905, + "loss/value_avg": 0.2414318025112152, + "lr": 3.2231333591650567e-06, + "objective/entropy": -64.31414031982422, + "objective/kl": 46.58558654785156, + "objective/non_score_reward": -2.3292794227600098, + "objective/rlhf_reward": 14.576970100402832, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009546708315610886, + "policy/clipfrac_avg": 0.00552853848785162, + "policy/entropy_avg": 0.189300537109375, + "step": 787, + "val/clipfrac_avg": 0.0013757861452177167, + "val/num_eos_tokens": 363, + "val/ratio": 0.987205982208252, + "val/ratio_var": 3.5099023079965264e-05 + }, + { + "episode": 12608, + "epoch": 0.6367676767676768, + "eps": 0, + "loss/policy_avg": -0.19538097083568573, + "loss/value_avg": 0.7160595655441284, + "lr": 3.210654495497068e-06, + "objective/entropy": -57.81846237182617, + "objective/kl": 42.85956573486328, + "objective/non_score_reward": -2.1429784297943115, + "objective/rlhf_reward": 12.14999008178711, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0011135776294395328, + "policy/clipfrac_avg": 0.009378955699503422, + "policy/entropy_avg": 0.2034454345703125, + "step": 788, + "val/clipfrac_avg": 0.1690627932548523, + "val/num_eos_tokens": 481, + "val/ratio": 0.9861621856689453, + "val/ratio_var": 9.24188134376891e-05 + }, + { + "episode": 12624, + "epoch": 0.6375757575757576, + "eps": 0, + "loss/policy_avg": 0.048018310219049454, + "loss/value_avg": 0.4312187135219574, + "lr": 3.198188401480734e-06, + "objective/entropy": -76.67124938964844, + "objective/kl": 36.849369049072266, + "objective/non_score_reward": -1.8424686193466187, + "objective/rlhf_reward": 12.25128173828125, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009952555410563946, + "policy/clipfrac_avg": 0.004150367341935635, + "policy/entropy_avg": 0.18071389198303223, + "step": 789, + "val/clipfrac_avg": 0.08804323524236679, + "val/num_eos_tokens": 357, + "val/ratio": 0.9867403507232666, + "val/ratio_var": 0.00013288675108924508 + }, + { + "episode": 12640, + "epoch": 0.6383838383838384, + "eps": 0, + "loss/policy_avg": 0.18971526622772217, + "loss/value_avg": 0.6626036167144775, + "lr": 3.185735166080244e-06, + "objective/entropy": -126.28926086425781, + "objective/kl": 33.96638488769531, + "objective/non_score_reward": -1.6983189582824707, + "objective/rlhf_reward": 12.973556518554688, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0007302669691853225, + "policy/clipfrac_avg": 0.008397979661822319, + "policy/entropy_avg": 0.14786386489868164, + "step": 790, + "val/clipfrac_avg": 0.06916166841983795, + "val/num_eos_tokens": 600, + "val/ratio": 0.9910507798194885, + "val/ratio_var": 5.283964492264204e-05 + }, + { + "episode": 12656, + "epoch": 0.6391919191919192, + "eps": 0, + "loss/policy_avg": -0.4279332160949707, + "loss/value_avg": 0.5117957592010498, + "lr": 3.173294878168025e-06, + "objective/entropy": -75.64164733886719, + "objective/kl": 40.16947555541992, + "objective/non_score_reward": -2.0084738731384277, + "objective/rlhf_reward": 12.86652660369873, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008897786610759795, + "policy/clipfrac_avg": 0.004540015943348408, + "policy/entropy_avg": 0.18616324663162231, + "step": 791, + "val/clipfrac_avg": 0.0338541679084301, + "val/num_eos_tokens": 224, + "val/ratio": 0.9873502254486084, + "val/ratio_var": 6.438903801608831e-05 + }, + { + "episode": 12672, + "epoch": 0.64, + "eps": 0, + "loss/policy_avg": -0.30223172903060913, + "loss/value_avg": 0.7414761185646057, + "lr": 3.160867626524107e-06, + "objective/entropy": -36.1086540222168, + "objective/kl": 35.08636474609375, + "objective/non_score_reward": -1.7543182373046875, + "objective/rlhf_reward": 14.995681762695312, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010164354462176561, + "policy/clipfrac_avg": 0.0033426620066165924, + "policy/entropy_avg": 0.20384597778320312, + "step": 792, + "val/clipfrac_avg": 0.08375494927167892, + "val/num_eos_tokens": 121, + "val/ratio": 0.9862762093544006, + "val/ratio_var": 8.216432615881786e-05 + }, + { + "episode": 12688, + "epoch": 0.6408080808080808, + "eps": 0, + "loss/policy_avg": 0.11556120216846466, + "loss/value_avg": 0.44696569442749023, + "lr": 3.1484534998354832e-06, + "objective/entropy": -52.89218521118164, + "objective/kl": 33.01978302001953, + "objective/non_score_reward": -1.650989294052124, + "objective/rlhf_reward": 12.270885467529297, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009637182811275125, + "policy/clipfrac_avg": 0.006595691200345755, + "policy/entropy_avg": 0.18551349639892578, + "step": 793, + "val/clipfrac_avg": 0.0313958078622818, + "val/num_eos_tokens": 483, + "val/ratio": 0.987555742263794, + "val/ratio_var": 4.9779242544900626e-05 + }, + { + "episode": 12704, + "epoch": 0.6416161616161616, + "eps": 0, + "loss/policy_avg": -0.11985203623771667, + "loss/value_avg": 0.5595318078994751, + "lr": 3.136052586695481e-06, + "objective/entropy": -44.2855110168457, + "objective/kl": 38.74113845825195, + "objective/non_score_reward": -1.9370570182800293, + "objective/rlhf_reward": 13.531692504882812, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009892593370750546, + "policy/clipfrac_avg": 0.006002710200846195, + "policy/entropy_avg": 0.20067405700683594, + "step": 794, + "val/clipfrac_avg": 0.051367055624723434, + "val/num_eos_tokens": 146, + "val/ratio": 0.98707115650177, + "val/ratio_var": 6.210069841472432e-05 + }, + { + "episode": 12720, + "epoch": 0.6424242424242425, + "eps": 0, + "loss/policy_avg": 0.06161116436123848, + "loss/value_avg": 0.20387178659439087, + "lr": 3.12366497560313e-06, + "objective/entropy": -47.063812255859375, + "objective/kl": 36.60810852050781, + "objective/non_score_reward": -1.8304054737091064, + "objective/rlhf_reward": 13.661782264709473, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000998336123302579, + "policy/clipfrac_avg": 0.004183639772236347, + "policy/entropy_avg": 0.19140625, + "step": 795, + "val/clipfrac_avg": 0.08834637701511383, + "val/num_eos_tokens": 342, + "val/ratio": 0.9874628782272339, + "val/ratio_var": 3.3792719477787614e-05 + }, + { + "episode": 12736, + "epoch": 0.6432323232323233, + "eps": 0, + "loss/policy_avg": 0.02652081847190857, + "loss/value_avg": 0.4432865381240845, + "lr": 3.1112907549625304e-06, + "objective/entropy": -104.79203796386719, + "objective/kl": 35.12114715576172, + "objective/non_score_reward": -1.7560575008392334, + "objective/rlhf_reward": 13.486129760742188, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008146626059897244, + "policy/clipfrac_avg": 0.0056906514801084995, + "policy/entropy_avg": 0.1502237319946289, + "step": 796, + "val/clipfrac_avg": 0.061473242938518524, + "val/num_eos_tokens": 748, + "val/ratio": 0.9898946285247803, + "val/ratio_var": 4.619114042725414e-05 + }, + { + "episode": 12752, + "epoch": 0.6440404040404041, + "eps": 0, + "loss/policy_avg": 0.06834741681814194, + "loss/value_avg": 0.5731651782989502, + "lr": 3.098930013082223e-06, + "objective/entropy": -88.31727600097656, + "objective/kl": 38.224117279052734, + "objective/non_score_reward": -1.9112060070037842, + "objective/rlhf_reward": 14.698169708251953, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008200570591725409, + "policy/clipfrac_avg": 0.007693252060562372, + "policy/entropy_avg": 0.1607208251953125, + "step": 797, + "val/clipfrac_avg": 0.11237858235836029, + "val/num_eos_tokens": 240, + "val/ratio": 0.9905504584312439, + "val/ratio_var": 4.706854815594852e-05 + }, + { + "episode": 12768, + "epoch": 0.6448484848484849, + "eps": 0, + "loss/policy_avg": -0.09761235117912292, + "loss/value_avg": 0.7088909149169922, + "lr": 3.0865828381745515e-06, + "objective/entropy": -124.7913589477539, + "objective/kl": 29.702392578125, + "objective/non_score_reward": -1.4851195812225342, + "objective/rlhf_reward": 13.68675422668457, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008782441727817059, + "policy/clipfrac_avg": 0.00598434591665864, + "policy/entropy_avg": 0.16924285888671875, + "step": 798, + "val/clipfrac_avg": 0.030641667544841766, + "val/num_eos_tokens": 454, + "val/ratio": 0.9882635474205017, + "val/ratio_var": 7.515518518630415e-05 + }, + { + "episode": 12784, + "epoch": 0.6456565656565657, + "eps": 0, + "loss/policy_avg": 0.14146611094474792, + "loss/value_avg": 0.7321012616157532, + "lr": 3.074249318355046e-06, + "objective/entropy": -101.5077133178711, + "objective/kl": 25.74894142150879, + "objective/non_score_reward": -1.2874470949172974, + "objective/rlhf_reward": 13.735990524291992, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007512210868299007, + "policy/clipfrac_avg": 0.011982694268226624, + "policy/entropy_avg": 0.15892601013183594, + "step": 799, + "val/clipfrac_avg": 0.012677992694079876, + "val/num_eos_tokens": 12, + "val/ratio": 0.9910134077072144, + "val/ratio_var": 6.09022390563041e-05 + }, + { + "episode": 12800, + "epoch": 0.6464646464646465, + "eps": 0, + "loss/policy_avg": -0.042851828038692474, + "loss/value_avg": 0.3452759385108948, + "lr": 3.0619295416417856e-06, + "objective/entropy": -68.88214874267578, + "objective/kl": 34.023399353027344, + "objective/non_score_reward": -1.7011699676513672, + "objective/rlhf_reward": 12.494142532348633, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010239251423627138, + "policy/clipfrac_avg": 0.010894561186432838, + "policy/entropy_avg": 0.17738628387451172, + "step": 800, + "val/clipfrac_avg": 0.026722928509116173, + "val/num_eos_tokens": 67, + "val/ratio": 0.9888595342636108, + "val/ratio_var": 6.888628558954224e-05 + }, + { + "episode": 12816, + "epoch": 0.6472727272727272, + "eps": 0, + "loss/policy_avg": -0.05754968896508217, + "loss/value_avg": 0.402332067489624, + "lr": 3.049623595954766e-06, + "objective/entropy": -89.62500762939453, + "objective/kl": 32.48891067504883, + "objective/non_score_reward": -1.6244455575942993, + "objective/rlhf_reward": 14.227116584777832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009385538287460804, + "policy/clipfrac_avg": 0.004920909181237221, + "policy/entropy_avg": 0.17440450191497803, + "step": 801, + "val/clipfrac_avg": 0.03176930546760559, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880223274230957, + "val/ratio_var": 5.388163845054805e-05 + }, + { + "episode": 12832, + "epoch": 0.648080808080808, + "eps": 0, + "loss/policy_avg": 0.2483101636171341, + "loss/value_avg": 0.23584425449371338, + "lr": 3.0373315691152856e-06, + "objective/entropy": -89.31193542480469, + "objective/kl": 40.699127197265625, + "objective/non_score_reward": -2.034956455230713, + "objective/rlhf_reward": 12.160355567932129, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0008679171442054212, + "policy/clipfrac_avg": 0.004822667222470045, + "policy/entropy_avg": 0.1660480499267578, + "step": 802, + "val/clipfrac_avg": 0.05813268944621086, + "val/num_eos_tokens": 149, + "val/ratio": 0.9890165328979492, + "val/ratio_var": 5.875104761798866e-05 + }, + { + "episode": 12848, + "epoch": 0.6488888888888888, + "eps": 0, + "loss/policy_avg": -0.10593897104263306, + "loss/value_avg": 0.5456434488296509, + "lr": 3.0250535488453077e-06, + "objective/entropy": -148.4324188232422, + "objective/kl": 38.51166915893555, + "objective/non_score_reward": -1.9255836009979248, + "objective/rlhf_reward": 11.918167114257812, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0007962615345604718, + "policy/clipfrac_avg": 0.0040261996909976006, + "policy/entropy_avg": 0.16094493865966797, + "step": 803, + "val/clipfrac_avg": 0.04293014109134674, + "val/num_eos_tokens": 331, + "val/ratio": 0.9893560409545898, + "val/ratio_var": 6.857667176518589e-05 + }, + { + "episode": 12864, + "epoch": 0.6496969696969697, + "eps": 0, + "loss/policy_avg": 0.06602153182029724, + "loss/value_avg": 0.2823004722595215, + "lr": 3.0127896227668367e-06, + "objective/entropy": -115.89801788330078, + "objective/kl": 40.921504974365234, + "objective/non_score_reward": -2.0460751056671143, + "objective/rlhf_reward": 12.867987632751465, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008664772612974048, + "policy/clipfrac_avg": 0.00966357346624136, + "policy/entropy_avg": 0.151214599609375, + "step": 804, + "val/clipfrac_avg": 0.03888742998242378, + "val/num_eos_tokens": 224, + "val/ratio": 0.9900601506233215, + "val/ratio_var": 5.00909227412194e-05 + }, + { + "episode": 12880, + "epoch": 0.6505050505050505, + "eps": 0, + "loss/policy_avg": -0.39892637729644775, + "loss/value_avg": 0.4240272641181946, + "lr": 3.000539878401296e-06, + "objective/entropy": -77.75199890136719, + "objective/kl": 35.056907653808594, + "objective/non_score_reward": -1.7528454065322876, + "objective/rlhf_reward": 14.00496768951416, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.001023960066959262, + "policy/clipfrac_avg": 0.004629913717508316, + "policy/entropy_avg": 0.20476627349853516, + "step": 805, + "val/clipfrac_avg": 0.07910472899675369, + "val/num_eos_tokens": 219, + "val/ratio": 0.9861233830451965, + "val/ratio_var": 8.373144373763353e-05 + }, + { + "episode": 12896, + "epoch": 0.6513131313131313, + "eps": 0, + "loss/policy_avg": -0.23323668539524078, + "loss/value_avg": 0.4607469439506531, + "lr": 2.9883044031688995e-06, + "objective/entropy": -85.76292419433594, + "objective/kl": 40.88092803955078, + "objective/non_score_reward": -2.044046401977539, + "objective/rlhf_reward": 14.174703598022461, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010081828804686666, + "policy/clipfrac_avg": 0.0041284579783678055, + "policy/entropy_avg": 0.1921253204345703, + "step": 806, + "val/clipfrac_avg": 0.16490784287452698, + "val/num_eos_tokens": 249, + "val/ratio": 0.9875838756561279, + "val/ratio_var": 6.450649379985407e-05 + }, + { + "episode": 12912, + "epoch": 0.6521212121212121, + "eps": 0, + "loss/policy_avg": -0.11845395714044571, + "loss/value_avg": 0.4592535197734833, + "lr": 2.976083284388031e-06, + "objective/entropy": -62.67930221557617, + "objective/kl": 33.47719955444336, + "objective/non_score_reward": -1.6738600730895996, + "objective/rlhf_reward": 12.771451950073242, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011091704946011305, + "policy/clipfrac_avg": 0.00747725460678339, + "policy/entropy_avg": 0.1983814239501953, + "step": 807, + "val/clipfrac_avg": 0.033114466816186905, + "val/num_eos_tokens": 17, + "val/ratio": 0.9877710938453674, + "val/ratio_var": 6.532426050398499e-05 + }, + { + "episode": 12928, + "epoch": 0.6529292929292929, + "eps": 0, + "loss/policy_avg": 0.0469900444149971, + "loss/value_avg": 0.26954370737075806, + "lr": 2.9638766092746207e-06, + "objective/entropy": -88.9464111328125, + "objective/kl": 34.25027847290039, + "objective/non_score_reward": -1.7125139236450195, + "objective/rlhf_reward": 12.56873607635498, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009448326891288161, + "policy/clipfrac_avg": 0.006387755274772644, + "policy/entropy_avg": 0.18566322326660156, + "step": 808, + "val/clipfrac_avg": 0.018177732825279236, + "val/num_eos_tokens": 761, + "val/ratio": 0.9882063269615173, + "val/ratio_var": 5.754105222877115e-05 + }, + { + "episode": 12944, + "epoch": 0.6537373737373737, + "eps": 0, + "loss/policy_avg": 0.00823296420276165, + "loss/value_avg": 0.44794195890426636, + "lr": 2.9516844649415165e-06, + "objective/entropy": -93.69944763183594, + "objective/kl": 40.86418151855469, + "objective/non_score_reward": -2.0432090759277344, + "objective/rlhf_reward": 13.542728424072266, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000799038156401366, + "policy/clipfrac_avg": 0.006408605724573135, + "policy/entropy_avg": 0.14969253540039062, + "step": 809, + "val/clipfrac_avg": 0.08976449072360992, + "val/num_eos_tokens": 215, + "val/ratio": 0.989780068397522, + "val/ratio_var": 6.0727201343979686e-05 + }, + { + "episode": 12960, + "epoch": 0.6545454545454545, + "eps": 0, + "loss/policy_avg": 0.024010583758354187, + "loss/value_avg": 0.3378852605819702, + "lr": 2.9395069383978725e-06, + "objective/entropy": -41.68758773803711, + "objective/kl": 35.73942184448242, + "objective/non_score_reward": -1.7869713306427002, + "objective/rlhf_reward": 15.314591407775879, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008846010314300656, + "policy/clipfrac_avg": 0.00692939106374979, + "policy/entropy_avg": 0.17317676544189453, + "step": 810, + "val/clipfrac_avg": 0.0032868448179215193, + "val/num_eos_tokens": 327, + "val/ratio": 0.9892865419387817, + "val/ratio_var": 4.3601932702586055e-05 + }, + { + "episode": 12976, + "epoch": 0.6553535353535354, + "eps": 0, + "loss/policy_avg": 0.1456170380115509, + "loss/value_avg": 0.5115708112716675, + "lr": 2.9273441165485227e-06, + "objective/entropy": -45.97090148925781, + "objective/kl": 36.01569366455078, + "objective/non_score_reward": -1.8007844686508179, + "objective/rlhf_reward": 14.23046588897705, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009762131376191974, + "policy/clipfrac_avg": 0.004803863354027271, + "policy/entropy_avg": 0.18830299377441406, + "step": 811, + "val/clipfrac_avg": 0.07713419198989868, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878898859024048, + "val/ratio_var": 4.7962184908101335e-05 + }, + { + "episode": 12992, + "epoch": 0.6561616161616162, + "eps": 0, + "loss/policy_avg": 0.1002146378159523, + "loss/value_avg": 0.558029294013977, + "lr": 2.9151960861933616e-06, + "objective/entropy": -66.14154052734375, + "objective/kl": 36.481346130371094, + "objective/non_score_reward": -1.8240673542022705, + "objective/rlhf_reward": 14.175932884216309, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009823704604059458, + "policy/clipfrac_avg": 0.0034461808390915394, + "policy/entropy_avg": 0.1955432891845703, + "step": 812, + "val/clipfrac_avg": 0.10748951137065887, + "val/num_eos_tokens": 292, + "val/ratio": 0.986059308052063, + "val/ratio_var": 5.328569022822194e-05 + }, + { + "episode": 13008, + "epoch": 0.656969696969697, + "eps": 0, + "loss/policy_avg": -0.10509809851646423, + "loss/value_avg": 0.46006062626838684, + "lr": 2.9030629340267165e-06, + "objective/entropy": -110.846923828125, + "objective/kl": 36.021034240722656, + "objective/non_score_reward": -1.8010516166687012, + "objective/rlhf_reward": 16.33176040649414, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008035163627937436, + "policy/clipfrac_avg": 0.005940265022218227, + "policy/entropy_avg": 0.15110206604003906, + "step": 813, + "val/clipfrac_avg": 0.07604548335075378, + "val/num_eos_tokens": 523, + "val/ratio": 0.9898133277893066, + "val/ratio_var": 6.0903592384420335e-05 + }, + { + "episode": 13024, + "epoch": 0.6577777777777778, + "eps": 0, + "loss/policy_avg": -0.11711955070495605, + "loss/value_avg": 1.2142528295516968, + "lr": 2.890944746636748e-06, + "objective/entropy": -40.75424575805664, + "objective/kl": 34.03726577758789, + "objective/non_score_reward": -1.701863408088684, + "objective/rlhf_reward": 13.091104507446289, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.000980376498773694, + "policy/clipfrac_avg": 0.008426186628639698, + "policy/entropy_avg": 0.20235157012939453, + "step": 814, + "val/clipfrac_avg": 0.02976190485060215, + "val/num_eos_tokens": 113, + "val/ratio": 0.9871314764022827, + "val/ratio_var": 6.27562912995927e-05 + }, + { + "episode": 13040, + "epoch": 0.6585858585858586, + "eps": 0, + "loss/policy_avg": -0.18143659830093384, + "loss/value_avg": 0.571576714515686, + "lr": 2.8788416105048124e-06, + "objective/entropy": -141.41366577148438, + "objective/kl": 42.53788757324219, + "objective/non_score_reward": -2.126894474029541, + "objective/rlhf_reward": 13.537168502807617, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008768361294642091, + "policy/clipfrac_avg": 0.006732264067977667, + "policy/entropy_avg": 0.17724037170410156, + "step": 815, + "val/clipfrac_avg": 0.03276197984814644, + "val/num_eos_tokens": 221, + "val/ratio": 0.9885480403900146, + "val/ratio_var": 8.229191007558256e-05 + }, + { + "episode": 13056, + "epoch": 0.6593939393939394, + "eps": 0, + "loss/policy_avg": 0.3017633855342865, + "loss/value_avg": 0.32543864846229553, + "lr": 2.8667536120048616e-06, + "objective/entropy": -61.10264587402344, + "objective/kl": 35.6143684387207, + "objective/non_score_reward": -1.7807183265686035, + "objective/rlhf_reward": 14.391157150268555, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009194887243211269, + "policy/clipfrac_avg": 0.018624408170580864, + "policy/entropy_avg": 0.14903783798217773, + "step": 816, + "val/clipfrac_avg": 0.0073128631338477135, + "val/num_eos_tokens": 112, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.833535033161752e-05 + }, + { + "episode": 13072, + "epoch": 0.6602020202020202, + "eps": 0, + "loss/policy_avg": 0.06899265199899673, + "loss/value_avg": 0.6989455223083496, + "lr": 2.854680837402804e-06, + "objective/entropy": -117.86632537841797, + "objective/kl": 34.15174102783203, + "objective/non_score_reward": -1.7075870037078857, + "objective/rlhf_reward": 14.511162757873535, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009505614871159196, + "policy/clipfrac_avg": 0.008451011963188648, + "policy/entropy_avg": 0.1699972152709961, + "step": 817, + "val/clipfrac_avg": 0.000589622650295496, + "val/num_eos_tokens": 435, + "val/ratio": 0.9876314401626587, + "val/ratio_var": 9.73327478277497e-05 + }, + { + "episode": 13088, + "epoch": 0.661010101010101, + "eps": 0, + "loss/policy_avg": -0.05428570508956909, + "loss/value_avg": 0.2928343415260315, + "lr": 2.8426233728559172e-06, + "objective/entropy": -52.28473663330078, + "objective/kl": 40.512550354003906, + "objective/non_score_reward": -2.025627613067627, + "objective/rlhf_reward": 14.458747863769531, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008845083648338914, + "policy/clipfrac_avg": 0.005094105843454599, + "policy/entropy_avg": 0.16685867309570312, + "step": 818, + "val/clipfrac_avg": 0.09271319210529327, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885064959526062, + "val/ratio_var": 4.853576319874264e-05 + }, + { + "episode": 13104, + "epoch": 0.6618181818181819, + "eps": 0, + "loss/policy_avg": 0.12085426598787308, + "loss/value_avg": 0.5670195817947388, + "lr": 2.83058130441221e-06, + "objective/entropy": -196.3238983154297, + "objective/kl": 42.012672424316406, + "objective/non_score_reward": -2.1006336212158203, + "objective/rlhf_reward": 12.51655387878418, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007904545636847615, + "policy/clipfrac_avg": 0.006419865880161524, + "policy/entropy_avg": 0.1437664031982422, + "step": 819, + "val/clipfrac_avg": 0.05388574302196503, + "val/num_eos_tokens": 496, + "val/ratio": 0.9899193644523621, + "val/ratio_var": 4.292546145734377e-05 + }, + { + "episode": 13120, + "epoch": 0.6626262626262627, + "eps": 0, + "loss/policy_avg": 0.6058844327926636, + "loss/value_avg": 0.7294787168502808, + "lr": 2.8185547180098195e-06, + "objective/entropy": -58.513553619384766, + "objective/kl": 38.73785400390625, + "objective/non_score_reward": -1.9368927478790283, + "objective/rlhf_reward": 12.883419036865234, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010288474150002003, + "policy/clipfrac_avg": 0.021498143672943115, + "policy/entropy_avg": 0.20896148681640625, + "step": 820, + "val/clipfrac_avg": 0.0723545029759407, + "val/num_eos_tokens": 134, + "val/ratio": 0.9887969493865967, + "val/ratio_var": 4.8135509132407606e-05 + }, + { + "episode": 13136, + "epoch": 0.6634343434343435, + "eps": 0, + "loss/policy_avg": -0.21932660043239594, + "loss/value_avg": 0.45023149251937866, + "lr": 2.8065436994763963e-06, + "objective/entropy": -38.43750762939453, + "objective/kl": 33.36050033569336, + "objective/non_score_reward": -1.6680251359939575, + "objective/rlhf_reward": 13.542912483215332, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010698122205212712, + "policy/clipfrac_avg": 0.004931025672703981, + "policy/entropy_avg": 0.19058465957641602, + "step": 821, + "val/clipfrac_avg": 0.00852490495890379, + "val/num_eos_tokens": 190, + "val/ratio": 0.9864101409912109, + "val/ratio_var": 5.521172352018766e-05 + }, + { + "episode": 13152, + "epoch": 0.6642424242424242, + "eps": 0, + "loss/policy_avg": -0.01703064516186714, + "loss/value_avg": 0.2902008295059204, + "lr": 2.794548334528486e-06, + "objective/entropy": 13.10763931274414, + "objective/kl": 44.57916259765625, + "objective/non_score_reward": -2.2289581298828125, + "objective/rlhf_reward": 13.364792823791504, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0014038155786693096, + "policy/clipfrac_avg": 0.006509849801659584, + "policy/entropy_avg": 0.26366424560546875, + "step": 822, + "val/clipfrac_avg": 0.03128276392817497, + "val/num_eos_tokens": 35, + "val/ratio": 0.9824488162994385, + "val/ratio_var": 4.7705398174002767e-05 + }, + { + "episode": 13168, + "epoch": 0.665050505050505, + "eps": 0, + "loss/policy_avg": -0.02819274365901947, + "loss/value_avg": 0.2861873507499695, + "lr": 2.782568708770933e-06, + "objective/entropy": -94.84840393066406, + "objective/kl": 42.75294494628906, + "objective/non_score_reward": -2.1376476287841797, + "objective/rlhf_reward": 12.26860237121582, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010491260327398777, + "policy/clipfrac_avg": 0.004985281266272068, + "policy/entropy_avg": 0.2000885009765625, + "step": 823, + "val/clipfrac_avg": 0.00754884909838438, + "val/num_eos_tokens": 289, + "val/ratio": 0.9866741895675659, + "val/ratio_var": 4.918476770399138e-05 + }, + { + "episode": 13184, + "epoch": 0.6658585858585858, + "eps": 0, + "loss/policy_avg": -0.024818487465381622, + "loss/value_avg": 0.3611629605293274, + "lr": 2.7706049076962483e-06, + "objective/entropy": -104.7535400390625, + "objective/kl": 45.08838653564453, + "objective/non_score_reward": -2.2544193267822266, + "objective/rlhf_reward": 12.214330673217773, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009155337465927005, + "policy/clipfrac_avg": 0.008831838145852089, + "policy/entropy_avg": 0.16748523712158203, + "step": 824, + "val/clipfrac_avg": 0.0024258445482701063, + "val/num_eos_tokens": 163, + "val/ratio": 0.9890375137329102, + "val/ratio_var": 5.131318175699562e-05 + }, + { + "episode": 13200, + "epoch": 0.6666666666666666, + "eps": 0, + "loss/policy_avg": -0.28224268555641174, + "loss/value_avg": 0.8720104694366455, + "lr": 2.7586570166840154e-06, + "objective/entropy": -74.55836486816406, + "objective/kl": 30.59856605529785, + "objective/non_score_reward": -1.52992844581604, + "objective/rlhf_reward": 15.329446792602539, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008330290438607335, + "policy/clipfrac_avg": 0.010464916005730629, + "policy/entropy_avg": 0.15100550651550293, + "step": 825, + "val/clipfrac_avg": 0.08580771088600159, + "val/num_eos_tokens": 460, + "val/ratio": 0.9896235466003418, + "val/ratio_var": 9.328931628260761e-05 + }, + { + "episode": 13216, + "epoch": 0.6674747474747474, + "eps": 0, + "loss/policy_avg": -0.1715128868818283, + "loss/value_avg": 0.4540337920188904, + "lr": 2.746725121000273e-06, + "objective/entropy": -43.89616394042969, + "objective/kl": 41.760223388671875, + "objective/non_score_reward": -2.0880112648010254, + "objective/rlhf_reward": 13.872926712036133, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010317859705537558, + "policy/clipfrac_avg": 0.0054151020012795925, + "policy/entropy_avg": 0.19500446319580078, + "step": 826, + "val/clipfrac_avg": 0.03631855547428131, + "val/num_eos_tokens": 334, + "val/ratio": 0.9874616265296936, + "val/ratio_var": 6.214805034687743e-05 + }, + { + "episode": 13232, + "epoch": 0.6682828282828283, + "eps": 0, + "loss/policy_avg": -0.077702596783638, + "loss/value_avg": 0.6576616168022156, + "lr": 2.734809305796915e-06, + "objective/entropy": -13.071683883666992, + "objective/kl": 34.05264663696289, + "objective/non_score_reward": -1.7026324272155762, + "objective/rlhf_reward": 14.656742095947266, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0012533586705103517, + "policy/clipfrac_avg": 0.00517519935965538, + "policy/entropy_avg": 0.21735763549804688, + "step": 827, + "val/clipfrac_avg": 0.09026823937892914, + "val/num_eos_tokens": 468, + "val/ratio": 0.9863014221191406, + "val/ratio_var": 0.00010488701082067564 + }, + { + "episode": 13248, + "epoch": 0.6690909090909091, + "eps": 0, + "loss/policy_avg": -0.11604561656713486, + "loss/value_avg": 0.18491807579994202, + "lr": 2.7229096561110703e-06, + "objective/entropy": -132.30560302734375, + "objective/kl": 37.0118522644043, + "objective/non_score_reward": -1.8505926132202148, + "objective/rlhf_reward": 15.282218933105469, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008336640894412994, + "policy/clipfrac_avg": 0.0030343476682901382, + "policy/entropy_avg": 0.16221046447753906, + "step": 828, + "val/clipfrac_avg": 0.07020965218544006, + "val/num_eos_tokens": 41, + "val/ratio": 0.9894965291023254, + "val/ratio_var": 4.505051037995145e-05 + }, + { + "episode": 13264, + "epoch": 0.6698989898989899, + "eps": 0, + "loss/policy_avg": -0.23082350194454193, + "loss/value_avg": 0.9338909387588501, + "lr": 2.7110262568645063e-06, + "objective/entropy": -83.5711441040039, + "objective/kl": 42.80398941040039, + "objective/non_score_reward": -2.1401994228363037, + "objective/rlhf_reward": 13.555112838745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008490884792990983, + "policy/clipfrac_avg": 0.024211544543504715, + "policy/entropy_avg": 0.16208171844482422, + "step": 829, + "val/clipfrac_avg": 0.01703277789056301, + "val/num_eos_tokens": 407, + "val/ratio": 0.9889751672744751, + "val/ratio_var": 6.549843965331092e-05 + }, + { + "episode": 13280, + "epoch": 0.6707070707070707, + "eps": 0, + "loss/policy_avg": -0.1790890097618103, + "loss/value_avg": 0.9402360320091248, + "lr": 2.699159192863018e-06, + "objective/entropy": -91.52725219726562, + "objective/kl": 30.28129005432129, + "objective/non_score_reward": -1.5140645503997803, + "objective/rlhf_reward": 13.71249771118164, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0006465939804911613, + "policy/clipfrac_avg": 0.003674568608403206, + "policy/entropy_avg": 0.14063799381256104, + "step": 830, + "val/clipfrac_avg": 0.05747377499938011, + "val/num_eos_tokens": 250, + "val/ratio": 0.9919025897979736, + "val/ratio_var": 6.961268809391186e-05 + }, + { + "episode": 13296, + "epoch": 0.6715151515151515, + "eps": 0, + "loss/policy_avg": 0.3400639295578003, + "loss/value_avg": 1.9688762426376343, + "lr": 2.687308548795825e-06, + "objective/entropy": -54.39606475830078, + "objective/kl": 44.12242889404297, + "objective/non_score_reward": -2.2061214447021484, + "objective/rlhf_reward": 14.004816055297852, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009418983245268464, + "policy/clipfrac_avg": 0.009523827582597733, + "policy/entropy_avg": 0.18965661525726318, + "step": 831, + "val/clipfrac_avg": 0.045346200466156006, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880996942520142, + "val/ratio_var": 6.565819057868794e-05 + }, + { + "episode": 13312, + "epoch": 0.6723232323232323, + "eps": 0, + "loss/policy_avg": 0.11609108746051788, + "loss/value_avg": 0.5635161399841309, + "lr": 2.675474409234965e-06, + "objective/entropy": -66.36386108398438, + "objective/kl": 38.93806838989258, + "objective/non_score_reward": -1.9469034671783447, + "objective/rlhf_reward": 14.076534271240234, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008638559957034886, + "policy/clipfrac_avg": 0.005322727840393782, + "policy/entropy_avg": 0.15152549743652344, + "step": 832, + "val/clipfrac_avg": 0.04381619766354561, + "val/num_eos_tokens": 20, + "val/ratio": 0.9889363050460815, + "val/ratio_var": 3.002676021424122e-05 + }, + { + "episode": 13328, + "epoch": 0.6731313131313131, + "eps": 0, + "loss/policy_avg": -0.03308400511741638, + "loss/value_avg": 0.3210276961326599, + "lr": 2.66365685863469e-06, + "objective/entropy": -111.39958953857422, + "objective/kl": 47.16798400878906, + "objective/non_score_reward": -2.3583993911743164, + "objective/rlhf_reward": 11.7197265625, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009453936363570392, + "policy/clipfrac_avg": 0.004603059496730566, + "policy/entropy_avg": 0.18264532089233398, + "step": 833, + "val/clipfrac_avg": 0.062541663646698, + "val/num_eos_tokens": 763, + "val/ratio": 0.9879401922225952, + "val/ratio_var": 6.487595965154469e-05 + }, + { + "episode": 13344, + "epoch": 0.673939393939394, + "eps": 0, + "loss/policy_avg": 0.051825061440467834, + "loss/value_avg": 0.8246160745620728, + "lr": 2.651855981330872e-06, + "objective/entropy": -69.66492462158203, + "objective/kl": 37.68748474121094, + "objective/non_score_reward": -1.8843741416931152, + "objective/rlhf_reward": 12.764063835144043, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010055399034172297, + "policy/clipfrac_avg": 0.004349449649453163, + "policy/entropy_avg": 0.18214893341064453, + "step": 834, + "val/clipfrac_avg": 0.06374436616897583, + "val/num_eos_tokens": 38, + "val/ratio": 0.9867125749588013, + "val/ratio_var": 9.083253826247528e-05 + }, + { + "episode": 13360, + "epoch": 0.6747474747474748, + "eps": 0, + "loss/policy_avg": -0.043558381497859955, + "loss/value_avg": 0.6093795895576477, + "lr": 2.6400718615403852e-06, + "objective/entropy": -108.79554748535156, + "objective/kl": 39.288482666015625, + "objective/non_score_reward": -1.9644241333007812, + "objective/rlhf_reward": 14.504325866699219, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010659170802682638, + "policy/clipfrac_avg": 0.005707671865820885, + "policy/entropy_avg": 0.1986398696899414, + "step": 835, + "val/clipfrac_avg": 0.013371335342526436, + "val/num_eos_tokens": 647, + "val/ratio": 0.9869523048400879, + "val/ratio_var": 8.708865061635152e-05 + }, + { + "episode": 13376, + "epoch": 0.6755555555555556, + "eps": 0, + "loss/policy_avg": 0.09040576219558716, + "loss/value_avg": 0.4517725706100464, + "lr": 2.62830458336052e-06, + "objective/entropy": -59.1908073425293, + "objective/kl": 42.06044387817383, + "objective/non_score_reward": -2.1030220985412598, + "objective/rlhf_reward": 13.842290878295898, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001016529044136405, + "policy/clipfrac_avg": 0.007354079745709896, + "policy/entropy_avg": 0.20728302001953125, + "step": 836, + "val/clipfrac_avg": 0.030187882483005524, + "val/num_eos_tokens": 477, + "val/ratio": 0.9865027666091919, + "val/ratio_var": 5.405073534348048e-05 + }, + { + "episode": 13392, + "epoch": 0.6763636363636364, + "eps": 0, + "loss/policy_avg": -0.21371892094612122, + "loss/value_avg": 0.26992350816726685, + "lr": 2.6165542307683744e-06, + "objective/entropy": -99.01640319824219, + "objective/kl": 43.636474609375, + "objective/non_score_reward": -2.18182373046875, + "objective/rlhf_reward": 13.08380126953125, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009176525054499507, + "policy/clipfrac_avg": 0.006078221835196018, + "policy/entropy_avg": 0.1714763641357422, + "step": 837, + "val/clipfrac_avg": 0.025651777163147926, + "val/num_eos_tokens": 453, + "val/ratio": 0.988213300704956, + "val/ratio_var": 6.253684114199132e-05 + }, + { + "episode": 13408, + "epoch": 0.6771717171717172, + "eps": 0, + "loss/policy_avg": 0.08225658535957336, + "loss/value_avg": 0.4106031656265259, + "lr": 2.604820887620254e-06, + "objective/entropy": -62.21269989013672, + "objective/kl": 40.305843353271484, + "objective/non_score_reward": -2.015292167663574, + "objective/rlhf_reward": 15.75814437866211, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0009111049002967775, + "policy/clipfrac_avg": 0.005017759278416634, + "policy/entropy_avg": 0.18220901489257812, + "step": 838, + "val/clipfrac_avg": 0.0028512999415397644, + "val/num_eos_tokens": 268, + "val/ratio": 0.9878619909286499, + "val/ratio_var": 3.237010969314724e-05 + }, + { + "episode": 13424, + "epoch": 0.677979797979798, + "eps": 0, + "loss/policy_avg": 0.01355072483420372, + "loss/value_avg": 0.4876829981803894, + "lr": 2.5931046376510875e-06, + "objective/entropy": -38.304931640625, + "objective/kl": 35.24858093261719, + "objective/non_score_reward": -1.7624289989471436, + "objective/rlhf_reward": 12.284445762634277, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0008708706009201705, + "policy/clipfrac_avg": 0.005640837829560041, + "policy/entropy_avg": 0.16802191734313965, + "step": 839, + "val/clipfrac_avg": 0.029899097979068756, + "val/num_eos_tokens": 258, + "val/ratio": 0.989201009273529, + "val/ratio_var": 5.1689257816178724e-05 + }, + { + "episode": 13440, + "epoch": 0.6787878787878788, + "eps": 0, + "loss/policy_avg": 0.1623387336730957, + "loss/value_avg": 0.3584904074668884, + "lr": 2.5814055644738013e-06, + "objective/entropy": -63.923397064208984, + "objective/kl": 36.323509216308594, + "objective/non_score_reward": -1.8161752223968506, + "objective/rlhf_reward": 12.97288703918457, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009589061955921352, + "policy/clipfrac_avg": 0.006033012643456459, + "policy/entropy_avg": 0.17229270935058594, + "step": 840, + "val/clipfrac_avg": 0.11569125950336456, + "val/num_eos_tokens": 159, + "val/ratio": 0.988390326499939, + "val/ratio_var": 7.233954966068268e-05 + }, + { + "episode": 13456, + "epoch": 0.6795959595959596, + "eps": 0, + "loss/policy_avg": -0.29571330547332764, + "loss/value_avg": 0.5451829433441162, + "lr": 2.5697237515787564e-06, + "objective/entropy": -159.0351104736328, + "objective/kl": 34.90948486328125, + "objective/non_score_reward": -1.7454744577407837, + "objective/rlhf_reward": 13.145150184631348, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0007017417810857296, + "policy/clipfrac_avg": 0.005396063905209303, + "policy/entropy_avg": 0.12675732374191284, + "step": 841, + "val/clipfrac_avg": 0.019056372344493866, + "val/num_eos_tokens": 508, + "val/ratio": 0.9914215207099915, + "val/ratio_var": 8.740482735447586e-05 + }, + { + "episode": 13472, + "epoch": 0.6804040404040405, + "eps": 0, + "loss/policy_avg": -0.33980700373649597, + "loss/value_avg": 0.5796597003936768, + "lr": 2.5580592823331237e-06, + "objective/entropy": -83.5593490600586, + "objective/kl": 36.281490325927734, + "objective/non_score_reward": -1.8140745162963867, + "objective/rlhf_reward": 14.18592643737793, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009474118705838919, + "policy/clipfrac_avg": 0.004876702558249235, + "policy/entropy_avg": 0.14034080505371094, + "step": 842, + "val/clipfrac_avg": 0.03151373565196991, + "val/num_eos_tokens": 208, + "val/ratio": 0.9901444911956787, + "val/ratio_var": 6.344315625028685e-05 + }, + { + "episode": 13488, + "epoch": 0.6812121212121212, + "eps": 0, + "loss/policy_avg": 0.11847849935293198, + "loss/value_avg": 0.3053319454193115, + "lr": 2.5464122399803126e-06, + "objective/entropy": -74.54005432128906, + "objective/kl": 46.23011779785156, + "objective/non_score_reward": -2.3115057945251465, + "objective/rlhf_reward": 12.993181228637695, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009884861065074801, + "policy/clipfrac_avg": 0.006667924579232931, + "policy/entropy_avg": 0.19658470153808594, + "step": 843, + "val/clipfrac_avg": 0.00010080645006382838, + "val/num_eos_tokens": 52, + "val/ratio": 0.9878820180892944, + "val/ratio_var": 4.166604412603192e-05 + }, + { + "episode": 13504, + "epoch": 0.682020202020202, + "eps": 0, + "loss/policy_avg": 0.07477365434169769, + "loss/value_avg": 0.717211902141571, + "lr": 2.5347827076393505e-06, + "objective/entropy": -55.136016845703125, + "objective/kl": 36.799835205078125, + "objective/non_score_reward": -1.8399916887283325, + "objective/rlhf_reward": 12.995945930480957, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010256943060085177, + "policy/clipfrac_avg": 0.009538116864860058, + "policy/entropy_avg": 0.19795989990234375, + "step": 844, + "val/clipfrac_avg": 0.0769520252943039, + "val/num_eos_tokens": 573, + "val/ratio": 0.9861817955970764, + "val/ratio_var": 7.031750283204019e-05 + }, + { + "episode": 13520, + "epoch": 0.6828282828282828, + "eps": 0, + "loss/policy_avg": -0.010380454361438751, + "loss/value_avg": 0.8978620767593384, + "lr": 2.5231707683043204e-06, + "objective/entropy": -164.994873046875, + "objective/kl": 41.09418487548828, + "objective/non_score_reward": -2.0547094345092773, + "objective/rlhf_reward": 12.367165565490723, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006862258305773139, + "policy/clipfrac_avg": 0.005633420776575804, + "policy/entropy_avg": 0.136183500289917, + "step": 845, + "val/clipfrac_avg": 0.10761284828186035, + "val/num_eos_tokens": 289, + "val/ratio": 0.9907996654510498, + "val/ratio_var": 4.169603198533878e-05 + }, + { + "episode": 13536, + "epoch": 0.6836363636363636, + "eps": 0, + "loss/policy_avg": -0.0011942088603973389, + "loss/value_avg": 0.7805696129798889, + "lr": 2.5115765048437445e-06, + "objective/entropy": -119.26876068115234, + "objective/kl": 37.73551940917969, + "objective/non_score_reward": -1.8867759704589844, + "objective/rlhf_reward": 13.769474029541016, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0007093651220202446, + "policy/clipfrac_avg": 0.028275154531002045, + "policy/entropy_avg": 0.1421506404876709, + "step": 846, + "val/clipfrac_avg": 0.08967090398073196, + "val/num_eos_tokens": 760, + "val/ratio": 0.9911968111991882, + "val/ratio_var": 3.498927253531292e-05 + }, + { + "episode": 13552, + "epoch": 0.6844444444444444, + "eps": 0, + "loss/policy_avg": -0.007019490003585815, + "loss/value_avg": 0.8198627233505249, + "lr": 2.5000000000000015e-06, + "objective/entropy": -89.728271484375, + "objective/kl": 34.76280975341797, + "objective/non_score_reward": -1.7381404638290405, + "objective/rlhf_reward": 13.925922393798828, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009024562314152718, + "policy/clipfrac_avg": 0.007501127198338509, + "policy/entropy_avg": 0.17042064666748047, + "step": 847, + "val/clipfrac_avg": 0.028372831642627716, + "val/num_eos_tokens": 17, + "val/ratio": 0.9899512529373169, + "val/ratio_var": 7.211479532998055e-05 + }, + { + "episode": 13568, + "epoch": 0.6852525252525252, + "eps": 0, + "loss/policy_avg": 0.015100933611392975, + "loss/value_avg": 0.25284868478775024, + "lr": 2.488441336388737e-06, + "objective/entropy": -184.20425415039062, + "objective/kl": 46.20475769042969, + "objective/non_score_reward": -2.3102383613586426, + "objective/rlhf_reward": 11.728824615478516, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007638686802238226, + "policy/clipfrac_avg": 0.005230149254202843, + "policy/entropy_avg": 0.15384864807128906, + "step": 848, + "val/clipfrac_avg": 0.01555095985531807, + "val/num_eos_tokens": 404, + "val/ratio": 0.9898365139961243, + "val/ratio_var": 5.412843165686354e-05 + }, + { + "episode": 13584, + "epoch": 0.686060606060606, + "eps": 0, + "loss/policy_avg": -0.13074953854084015, + "loss/value_avg": 0.5254482626914978, + "lr": 2.4769005964982718e-06, + "objective/entropy": -70.96700286865234, + "objective/kl": 37.31745147705078, + "objective/non_score_reward": -1.8658727407455444, + "objective/rlhf_reward": 13.665377616882324, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000885509536601603, + "policy/clipfrac_avg": 0.006637534126639366, + "policy/entropy_avg": 0.17204856872558594, + "step": 849, + "val/clipfrac_avg": 0.050514861941337585, + "val/num_eos_tokens": 24, + "val/ratio": 0.9882212281227112, + "val/ratio_var": 5.447021248983219e-05 + }, + { + "episode": 13600, + "epoch": 0.6868686868686869, + "eps": 0, + "loss/policy_avg": 0.039310187101364136, + "loss/value_avg": 0.18585552275180817, + "lr": 2.465377862689019e-06, + "objective/entropy": -62.67339324951172, + "objective/kl": 48.39326477050781, + "objective/non_score_reward": -2.419663190841675, + "objective/rlhf_reward": 13.564711570739746, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009431451326236129, + "policy/clipfrac_avg": 0.004864457994699478, + "policy/entropy_avg": 0.19204330444335938, + "step": 850, + "val/clipfrac_avg": 0.01945144683122635, + "val/num_eos_tokens": 36, + "val/ratio": 0.9872653484344482, + "val/ratio_var": 3.194732926203869e-05 + }, + { + "episode": 13616, + "epoch": 0.6876767676767677, + "eps": 0, + "loss/policy_avg": 0.07975710183382034, + "loss/value_avg": 0.478973388671875, + "lr": 2.4538732171928847e-06, + "objective/entropy": -76.03438568115234, + "objective/kl": 35.364593505859375, + "objective/non_score_reward": -1.7682294845581055, + "objective/rlhf_reward": 15.075519561767578, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008544789161533117, + "policy/clipfrac_avg": 0.005153194069862366, + "policy/entropy_avg": 0.16641998291015625, + "step": 851, + "val/clipfrac_avg": 0.03764945641160011, + "val/num_eos_tokens": 298, + "val/ratio": 0.9884979724884033, + "val/ratio_var": 4.7100515075726435e-05 + }, + { + "episode": 13632, + "epoch": 0.6884848484848485, + "eps": 0, + "loss/policy_avg": -0.10211583226919174, + "loss/value_avg": 0.7199238538742065, + "lr": 2.4423867421126923e-06, + "objective/entropy": -59.744590759277344, + "objective/kl": 35.15203857421875, + "objective/non_score_reward": -1.7576020956039429, + "objective/rlhf_reward": 15.05489730834961, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009704260155558586, + "policy/clipfrac_avg": 0.004688786808401346, + "policy/entropy_avg": 0.17905235290527344, + "step": 852, + "val/clipfrac_avg": 0.17230291664600372, + "val/num_eos_tokens": 644, + "val/ratio": 0.9872374534606934, + "val/ratio_var": 4.29930878453888e-05 + }, + { + "episode": 13648, + "epoch": 0.6892929292929293, + "eps": 0, + "loss/policy_avg": 0.0028443392366170883, + "loss/value_avg": 0.3421381115913391, + "lr": 2.4309185194215885e-06, + "objective/entropy": -63.859352111816406, + "objective/kl": 41.87160110473633, + "objective/non_score_reward": -2.0935800075531006, + "objective/rlhf_reward": 16.01579475402832, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0011341955978423357, + "policy/clipfrac_avg": 0.005558031145483255, + "policy/entropy_avg": 0.21439552307128906, + "step": 853, + "val/clipfrac_avg": 0.07718860357999802, + "val/num_eos_tokens": 400, + "val/ratio": 0.9858847856521606, + "val/ratio_var": 5.448433512356132e-05 + }, + { + "episode": 13664, + "epoch": 0.6901010101010101, + "eps": 0, + "loss/policy_avg": 0.11498575657606125, + "loss/value_avg": 0.909265398979187, + "lr": 2.4194686309624664e-06, + "objective/entropy": -138.73922729492188, + "objective/kl": 39.14732360839844, + "objective/non_score_reward": -1.9573662281036377, + "objective/rlhf_reward": 11.542634010314941, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0011413603788241744, + "policy/clipfrac_avg": 0.008165586739778519, + "policy/entropy_avg": 0.18193864822387695, + "step": 854, + "val/clipfrac_avg": 0.04511754959821701, + "val/num_eos_tokens": 264, + "val/ratio": 0.9880124926567078, + "val/ratio_var": 7.85917291068472e-05 + }, + { + "episode": 13680, + "epoch": 0.6909090909090909, + "eps": 0, + "loss/policy_avg": 0.4261026978492737, + "loss/value_avg": 1.0351085662841797, + "lr": 2.408037158447375e-06, + "objective/entropy": -80.8604736328125, + "objective/kl": 39.87467956542969, + "objective/non_score_reward": -1.9937340021133423, + "objective/rlhf_reward": 12.850015640258789, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000854466634336859, + "policy/clipfrac_avg": 0.0031361770816147327, + "policy/entropy_avg": 0.1807117462158203, + "step": 855, + "val/clipfrac_avg": 0.06461638957262039, + "val/num_eos_tokens": 76, + "val/ratio": 0.9892903566360474, + "val/ratio_var": 6.158453470561653e-05 + }, + { + "episode": 13696, + "epoch": 0.6917171717171717, + "eps": 0, + "loss/policy_avg": 0.35515934228897095, + "loss/value_avg": 0.6004530787467957, + "lr": 2.3966241834569364e-06, + "objective/entropy": -109.00687408447266, + "objective/kl": 44.59175109863281, + "objective/non_score_reward": -2.2295875549316406, + "objective/rlhf_reward": 14.02041244506836, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008301680209115148, + "policy/clipfrac_avg": 0.004295141436159611, + "policy/entropy_avg": 0.16374778747558594, + "step": 856, + "val/clipfrac_avg": 0.0578775517642498, + "val/num_eos_tokens": 355, + "val/ratio": 0.9892109632492065, + "val/ratio_var": 3.649172867881134e-05 + }, + { + "episode": 13712, + "epoch": 0.6925252525252525, + "eps": 0, + "loss/policy_avg": -0.07683120667934418, + "loss/value_avg": 0.7507062554359436, + "lr": 2.385229787439768e-06, + "objective/entropy": -65.75347137451172, + "objective/kl": 33.233463287353516, + "objective/non_score_reward": -1.6616731882095337, + "objective/rlhf_reward": 12.885202407836914, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008445363491773605, + "policy/clipfrac_avg": 0.0028887931257486343, + "policy/entropy_avg": 0.1688997745513916, + "step": 857, + "val/clipfrac_avg": 0.029584139585494995, + "val/num_eos_tokens": 79, + "val/ratio": 0.9893004894256592, + "val/ratio_var": 4.4566000724444166e-05 + }, + { + "episode": 13728, + "epoch": 0.6933333333333334, + "eps": 0, + "loss/policy_avg": -0.04102202132344246, + "loss/value_avg": 0.295141339302063, + "lr": 2.3738540517118953e-06, + "objective/entropy": -137.21546936035156, + "objective/kl": 39.995941162109375, + "objective/non_score_reward": -1.9997971057891846, + "objective/rlhf_reward": 12.875202178955078, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009317261283285916, + "policy/clipfrac_avg": 0.016368409618735313, + "policy/entropy_avg": 0.18145465850830078, + "step": 858, + "val/clipfrac_avg": 0.02603679709136486, + "val/num_eos_tokens": 133, + "val/ratio": 0.9879333972930908, + "val/ratio_var": 7.448039104929194e-05 + }, + { + "episode": 13744, + "epoch": 0.6941414141414142, + "eps": 0, + "loss/policy_avg": -0.09325989335775375, + "loss/value_avg": 0.4778023660182953, + "lr": 2.3624970574561773e-06, + "objective/entropy": -126.39140319824219, + "objective/kl": 41.33527755737305, + "objective/non_score_reward": -2.0667638778686523, + "objective/rlhf_reward": 15.823860168457031, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009032993111759424, + "policy/clipfrac_avg": 0.008262147195637226, + "policy/entropy_avg": 0.1809234619140625, + "step": 859, + "val/clipfrac_avg": 0.006358179729431868, + "val/num_eos_tokens": 198, + "val/ratio": 0.9881711602210999, + "val/ratio_var": 8.430143498117104e-05 + }, + { + "episode": 13760, + "epoch": 0.694949494949495, + "eps": 0, + "loss/policy_avg": -0.1992878019809723, + "loss/value_avg": 0.4450816214084625, + "lr": 2.3511588857217228e-06, + "objective/entropy": -82.32437133789062, + "objective/kl": 36.61549377441406, + "objective/non_score_reward": -1.8307745456695557, + "objective/rlhf_reward": 14.497350692749023, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009405683958902955, + "policy/clipfrac_avg": 0.0031182412058115005, + "policy/entropy_avg": 0.1852397918701172, + "step": 860, + "val/clipfrac_avg": 0.04030741751194, + "val/num_eos_tokens": 674, + "val/ratio": 0.9874944686889648, + "val/ratio_var": 5.9595575294224545e-05 + }, + { + "episode": 13776, + "epoch": 0.6957575757575758, + "eps": 0, + "loss/policy_avg": -0.0025226399302482605, + "loss/value_avg": 0.6712653636932373, + "lr": 2.339839617423318e-06, + "objective/entropy": -97.76516723632812, + "objective/kl": 32.36253356933594, + "objective/non_score_reward": -1.6181267499923706, + "objective/rlhf_reward": 14.108436584472656, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008775125024840236, + "policy/clipfrac_avg": 0.006763250567018986, + "policy/entropy_avg": 0.15996170043945312, + "step": 861, + "val/clipfrac_avg": 0.01572282984852791, + "val/num_eos_tokens": 755, + "val/ratio": 0.9897935390472412, + "val/ratio_var": 4.971764428773895e-05 + }, + { + "episode": 13792, + "epoch": 0.6965656565656566, + "eps": 0, + "loss/policy_avg": 0.08292508125305176, + "loss/value_avg": 0.5537270307540894, + "lr": 2.32853933334084e-06, + "objective/entropy": -61.261680603027344, + "objective/kl": 36.61561584472656, + "objective/non_score_reward": -1.8307808637619019, + "objective/rlhf_reward": 12.762969970703125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009761417168192565, + "policy/clipfrac_avg": 0.0026826506946235895, + "policy/entropy_avg": 0.22123146057128906, + "step": 862, + "val/clipfrac_avg": 0.044523607939481735, + "val/num_eos_tokens": 56, + "val/ratio": 0.9871484041213989, + "val/ratio_var": 7.335093687288463e-05 + }, + { + "episode": 13808, + "epoch": 0.6973737373737374, + "eps": 0, + "loss/policy_avg": 0.36910420656204224, + "loss/value_avg": 0.9228214025497437, + "lr": 2.317258114118686e-06, + "objective/entropy": -76.75859069824219, + "objective/kl": 36.92033004760742, + "objective/non_score_reward": -1.8460166454315186, + "objective/rlhf_reward": 14.872733116149902, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009935328271239996, + "policy/clipfrac_avg": 0.006773949135094881, + "policy/entropy_avg": 0.1806344985961914, + "step": 863, + "val/clipfrac_avg": 0.07717607915401459, + "val/num_eos_tokens": 514, + "val/ratio": 0.9876723289489746, + "val/ratio_var": 6.279607623582706e-05 + }, + { + "episode": 13824, + "epoch": 0.6981818181818182, + "eps": 0, + "loss/policy_avg": -0.21535149216651917, + "loss/value_avg": 0.42513978481292725, + "lr": 2.305996040265198e-06, + "objective/entropy": -77.51583862304688, + "objective/kl": 45.520545959472656, + "objective/non_score_reward": -2.276027202606201, + "objective/rlhf_reward": 14.59897232055664, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008458065567538142, + "policy/clipfrac_avg": 0.005439429543912411, + "policy/entropy_avg": 0.16998863220214844, + "step": 864, + "val/clipfrac_avg": 0.03808614984154701, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894691109657288, + "val/ratio_var": 6.017706618877128e-05 + }, + { + "episode": 13840, + "epoch": 0.6989898989898989, + "eps": 0, + "loss/policy_avg": -0.08341407775878906, + "loss/value_avg": 0.7828294038772583, + "lr": 2.29475319215209e-06, + "objective/entropy": -82.54826354980469, + "objective/kl": 39.21794891357422, + "objective/non_score_reward": -1.9608973264694214, + "objective/rlhf_reward": 14.359415054321289, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007373163243755698, + "policy/clipfrac_avg": 0.003970882389694452, + "policy/entropy_avg": 0.14882469177246094, + "step": 865, + "val/clipfrac_avg": 0.17675819993019104, + "val/num_eos_tokens": 134, + "val/ratio": 0.9905297756195068, + "val/ratio_var": 5.841355596203357e-05 + }, + { + "episode": 13856, + "epoch": 0.6997979797979798, + "eps": 0, + "loss/policy_avg": 0.029253050684928894, + "loss/value_avg": 0.8540998101234436, + "lr": 2.2835296500138664e-06, + "objective/entropy": -121.09165954589844, + "objective/kl": 34.1153564453125, + "objective/non_score_reward": -1.7057678699493408, + "objective/rlhf_reward": 13.333294868469238, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009264532127417624, + "policy/clipfrac_avg": 0.0050398847088217735, + "policy/entropy_avg": 0.18099594116210938, + "step": 866, + "val/clipfrac_avg": 0.02329857461154461, + "val/num_eos_tokens": 1353, + "val/ratio": 0.9883856177330017, + "val/ratio_var": 8.088548202067614e-05 + }, + { + "episode": 13872, + "epoch": 0.7006060606060606, + "eps": 0, + "loss/policy_avg": -0.1498204916715622, + "loss/value_avg": 0.4047302007675171, + "lr": 2.272325493947257e-06, + "objective/entropy": -49.65830993652344, + "objective/kl": 36.36531066894531, + "objective/non_score_reward": -1.8182655572891235, + "objective/rlhf_reward": 14.494234085083008, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010429773246869445, + "policy/clipfrac_avg": 0.005474097095429897, + "policy/entropy_avg": 0.18477344512939453, + "step": 867, + "val/clipfrac_avg": 0.0032526040449738503, + "val/num_eos_tokens": 101, + "val/ratio": 0.9877700805664062, + "val/ratio_var": 9.155944280792028e-05 + }, + { + "episode": 13888, + "epoch": 0.7014141414141414, + "eps": 0, + "loss/policy_avg": -0.1165279895067215, + "loss/value_avg": 0.7669879198074341, + "lr": 2.261140803910644e-06, + "objective/entropy": -77.34773254394531, + "objective/kl": 38.648643493652344, + "objective/non_score_reward": -1.9324320554733276, + "objective/rlhf_reward": 13.637880325317383, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000937658129259944, + "policy/clipfrac_avg": 0.006235727574676275, + "policy/entropy_avg": 0.19645977020263672, + "step": 868, + "val/clipfrac_avg": 0.058441728353500366, + "val/num_eos_tokens": 310, + "val/ratio": 0.9879204034805298, + "val/ratio_var": 6.414015660993755e-05 + }, + { + "episode": 13904, + "epoch": 0.7022222222222222, + "eps": 0, + "loss/policy_avg": -0.016327757388353348, + "loss/value_avg": 0.43844515085220337, + "lr": 2.2499756597234872e-06, + "objective/entropy": -163.28765869140625, + "objective/kl": 48.81157684326172, + "objective/non_score_reward": -2.4405789375305176, + "objective/rlhf_reward": 11.70395278930664, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.000844982685521245, + "policy/clipfrac_avg": 0.0032518147490918636, + "policy/entropy_avg": 0.16161727905273438, + "step": 869, + "val/clipfrac_avg": 0.014801801182329655, + "val/num_eos_tokens": 1949, + "val/ratio": 0.9885438680648804, + "val/ratio_var": 5.214377597440034e-05 + }, + { + "episode": 13920, + "epoch": 0.703030303030303, + "eps": 0, + "loss/policy_avg": -0.18964827060699463, + "loss/value_avg": 0.5136280655860901, + "lr": 2.238830141065765e-06, + "objective/entropy": -90.82677459716797, + "objective/kl": 40.3735237121582, + "objective/non_score_reward": -2.0186760425567627, + "objective/rlhf_reward": 15.153199195861816, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008435281342826784, + "policy/clipfrac_avg": 0.005789198447018862, + "policy/entropy_avg": 0.1680126190185547, + "step": 870, + "val/clipfrac_avg": 0.0648677870631218, + "val/num_eos_tokens": 555, + "val/ratio": 0.9890276193618774, + "val/ratio_var": 7.125848060240969e-05 + }, + { + "episode": 13936, + "epoch": 0.7038383838383838, + "eps": 0, + "loss/policy_avg": 0.02698252536356449, + "loss/value_avg": 0.2791609764099121, + "lr": 2.2277043274773856e-06, + "objective/entropy": -45.07421112060547, + "objective/kl": 39.21965026855469, + "objective/non_score_reward": -1.9609825611114502, + "objective/rlhf_reward": 14.624954223632812, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009133699350059032, + "policy/clipfrac_avg": 0.004030972719192505, + "policy/entropy_avg": 0.18420028686523438, + "step": 871, + "val/clipfrac_avg": 0.03648557886481285, + "val/num_eos_tokens": 232, + "val/ratio": 0.9881106615066528, + "val/ratio_var": 5.0894170271931216e-05 + }, + { + "episode": 13952, + "epoch": 0.7046464646464646, + "eps": 0, + "loss/policy_avg": -0.12599462270736694, + "loss/value_avg": 0.5007761716842651, + "lr": 2.216598298357646e-06, + "objective/entropy": -52.43424987792969, + "objective/kl": 37.084686279296875, + "objective/non_score_reward": -1.8542344570159912, + "objective/rlhf_reward": 14.18482780456543, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008421384263783693, + "policy/clipfrac_avg": 0.002664695493876934, + "policy/entropy_avg": 0.1716923713684082, + "step": 872, + "val/clipfrac_avg": 0.0010017228778451681, + "val/num_eos_tokens": 274, + "val/ratio": 0.9881088137626648, + "val/ratio_var": 6.610793934669346e-05 + }, + { + "episode": 13968, + "epoch": 0.7054545454545454, + "eps": 0, + "loss/policy_avg": 0.35178622603416443, + "loss/value_avg": 0.5846590995788574, + "lr": 2.2055121329646416e-06, + "objective/entropy": -138.92434692382812, + "objective/kl": 41.75783157348633, + "objective/non_score_reward": -2.0878915786743164, + "objective/rlhf_reward": 13.091796875, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008433774346485734, + "policy/clipfrac_avg": 0.006052501033991575, + "policy/entropy_avg": 0.16450119018554688, + "step": 873, + "val/clipfrac_avg": 0.0322103314101696, + "val/num_eos_tokens": 1144, + "val/ratio": 0.9888837337493896, + "val/ratio_var": 5.522933497559279e-05 + }, + { + "episode": 13984, + "epoch": 0.7062626262626263, + "eps": 0, + "loss/policy_avg": -0.004525661468505859, + "loss/value_avg": 0.5259770750999451, + "lr": 2.1944459104147188e-06, + "objective/entropy": -122.14696502685547, + "objective/kl": 37.81830978393555, + "objective/non_score_reward": -1.8909155130386353, + "objective/rlhf_reward": 13.296585083007812, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0006640740903094411, + "policy/clipfrac_avg": 0.004708480089902878, + "policy/entropy_avg": 0.14041614532470703, + "step": 874, + "val/clipfrac_avg": 0.10728074610233307, + "val/num_eos_tokens": 35, + "val/ratio": 0.990477979183197, + "val/ratio_var": 6.866270996397361e-05 + }, + { + "episode": 14000, + "epoch": 0.7070707070707071, + "eps": 0, + "loss/policy_avg": 0.11862924695014954, + "loss/value_avg": 1.0722770690917969, + "lr": 2.1833997096818897e-06, + "objective/entropy": -54.207332611083984, + "objective/kl": 37.62938690185547, + "objective/non_score_reward": -1.8814692497253418, + "objective/rlhf_reward": 14.2200927734375, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000859625346492976, + "policy/clipfrac_avg": 0.005390159785747528, + "policy/entropy_avg": 0.1755199432373047, + "step": 875, + "val/clipfrac_avg": 0.0593750923871994, + "val/num_eos_tokens": 201, + "val/ratio": 0.9878208637237549, + "val/ratio_var": 6.484645564341918e-05 + }, + { + "episode": 14016, + "epoch": 0.7078787878787879, + "eps": 0, + "loss/policy_avg": -0.2030085027217865, + "loss/value_avg": 0.5594953298568726, + "lr": 2.1723736095972946e-06, + "objective/entropy": -16.795242309570312, + "objective/kl": 44.697776794433594, + "objective/non_score_reward": -2.234889030456543, + "objective/rlhf_reward": 13.140110969543457, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0011914006900042295, + "policy/clipfrac_avg": 0.005575358867645264, + "policy/entropy_avg": 0.2543792724609375, + "step": 876, + "val/clipfrac_avg": 0.02393021248281002, + "val/num_eos_tokens": 24, + "val/ratio": 0.9835648536682129, + "val/ratio_var": 0.00010361165914218873 + }, + { + "episode": 14032, + "epoch": 0.7086868686868687, + "eps": 0, + "loss/policy_avg": 0.05069049820303917, + "loss/value_avg": 0.6454532146453857, + "lr": 2.161367688848618e-06, + "objective/entropy": -105.85357666015625, + "objective/kl": 50.78942108154297, + "objective/non_score_reward": -2.53947114944458, + "objective/rlhf_reward": 13.077716827392578, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008386933477595448, + "policy/clipfrac_avg": 0.003330180887132883, + "policy/entropy_avg": 0.1841726303100586, + "step": 877, + "val/clipfrac_avg": 0.010819870047271252, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886084198951721, + "val/ratio_var": 5.4145024478202686e-05 + }, + { + "episode": 14048, + "epoch": 0.7094949494949495, + "eps": 0, + "loss/policy_avg": -0.32361558079719543, + "loss/value_avg": 0.4275820851325989, + "lr": 2.1503820259795357e-06, + "objective/entropy": -193.71432495117188, + "objective/kl": 41.63684844970703, + "objective/non_score_reward": -2.0818424224853516, + "objective/rlhf_reward": 14.261907577514648, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007008577231317759, + "policy/clipfrac_avg": 0.004103646147996187, + "policy/entropy_avg": 0.1445927619934082, + "step": 878, + "val/clipfrac_avg": 7.62939453125e-06, + "val/num_eos_tokens": 323, + "val/ratio": 0.9903497695922852, + "val/ratio_var": 3.22636442433577e-05 + }, + { + "episode": 14064, + "epoch": 0.7103030303030303, + "eps": 0, + "loss/policy_avg": -0.1645442247390747, + "loss/value_avg": 0.7777354717254639, + "lr": 2.139416699389153e-06, + "objective/entropy": -45.89549255371094, + "objective/kl": 34.282958984375, + "objective/non_score_reward": -1.7141478061676025, + "objective/rlhf_reward": 14.692102432250977, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007650890620425344, + "policy/clipfrac_avg": 0.00561137031763792, + "policy/entropy_avg": 0.15566253662109375, + "step": 879, + "val/clipfrac_avg": 0.10208053886890411, + "val/num_eos_tokens": 49, + "val/ratio": 0.9895084500312805, + "val/ratio_var": 3.380152702447958e-05 + }, + { + "episode": 14080, + "epoch": 0.7111111111111111, + "eps": 0, + "loss/policy_avg": -0.023053357377648354, + "loss/value_avg": 0.521371603012085, + "lr": 2.128471787331446e-06, + "objective/entropy": -50.96327209472656, + "objective/kl": 41.30862808227539, + "objective/non_score_reward": -2.0654313564300537, + "objective/rlhf_reward": 14.450193405151367, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009377336828038096, + "policy/clipfrac_avg": 0.00421791011467576, + "policy/entropy_avg": 0.17507076263427734, + "step": 880, + "val/clipfrac_avg": 0.04448426887392998, + "val/num_eos_tokens": 368, + "val/ratio": 0.9889349937438965, + "val/ratio_var": 3.734078927664086e-05 + }, + { + "episode": 14096, + "epoch": 0.711919191919192, + "eps": 0, + "loss/policy_avg": -0.07007278501987457, + "loss/value_avg": 0.2162141352891922, + "lr": 2.117547367914705e-06, + "objective/entropy": -69.7264175415039, + "objective/kl": 41.05116653442383, + "objective/non_score_reward": -2.052558422088623, + "objective/rlhf_reward": 14.830253601074219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.001040782080963254, + "policy/clipfrac_avg": 0.0033751465380191803, + "policy/entropy_avg": 0.20317459106445312, + "step": 881, + "val/clipfrac_avg": 0.03193364292383194, + "val/num_eos_tokens": 551, + "val/ratio": 0.9861003160476685, + "val/ratio_var": 8.361194340977818e-05 + }, + { + "episode": 14112, + "epoch": 0.7127272727272728, + "eps": 0, + "loss/policy_avg": -0.07847104221582413, + "loss/value_avg": 0.2657424807548523, + "lr": 2.1066435191009717e-06, + "objective/entropy": -86.86426544189453, + "objective/kl": 45.00962829589844, + "objective/non_score_reward": -2.250481367111206, + "objective/rlhf_reward": 13.913580894470215, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010606081923469901, + "policy/clipfrac_avg": 0.0069802612997591496, + "policy/entropy_avg": 0.20689773559570312, + "step": 882, + "val/clipfrac_avg": 0.038475796580314636, + "val/num_eos_tokens": 546, + "val/ratio": 0.985522449016571, + "val/ratio_var": 8.580441499361768e-05 + }, + { + "episode": 14128, + "epoch": 0.7135353535353536, + "eps": 0, + "loss/policy_avg": -0.1323227733373642, + "loss/value_avg": 0.21754738688468933, + "lr": 2.095760318705487e-06, + "objective/entropy": -109.19290161132812, + "objective/kl": 42.61073303222656, + "objective/non_score_reward": -2.1305365562438965, + "objective/rlhf_reward": 14.447587966918945, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010359378065913916, + "policy/clipfrac_avg": 0.006892787292599678, + "policy/entropy_avg": 0.20017433166503906, + "step": 883, + "val/clipfrac_avg": 0.010656789876520634, + "val/num_eos_tokens": 36, + "val/ratio": 0.9864689111709595, + "val/ratio_var": 4.844724026042968e-05 + }, + { + "episode": 14144, + "epoch": 0.7143434343434344, + "eps": 0, + "loss/policy_avg": -0.1507091522216797, + "loss/value_avg": 0.2665415108203888, + "lr": 2.0848978443961314e-06, + "objective/entropy": -119.52471923828125, + "objective/kl": 43.517696380615234, + "objective/non_score_reward": -2.17588472366333, + "objective/rlhf_reward": 14.300678253173828, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000938809709623456, + "policy/clipfrac_avg": 0.012131594121456146, + "policy/entropy_avg": 0.18040943145751953, + "step": 884, + "val/clipfrac_avg": 0.006075765937566757, + "val/num_eos_tokens": 620, + "val/ratio": 0.9873965978622437, + "val/ratio_var": 6.621635111514479e-05 + }, + { + "episode": 14160, + "epoch": 0.7151515151515152, + "eps": 0, + "loss/policy_avg": -0.028530046343803406, + "loss/value_avg": 0.434810072183609, + "lr": 2.074056173692881e-06, + "objective/entropy": -62.273380279541016, + "objective/kl": 45.77116394042969, + "objective/non_score_reward": -2.2885584831237793, + "objective/rlhf_reward": 14.617691040039062, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009718666551634669, + "policy/clipfrac_avg": 0.004924894310534, + "policy/entropy_avg": 0.20779800415039062, + "step": 885, + "val/clipfrac_avg": 0.036309417337179184, + "val/num_eos_tokens": 179, + "val/ratio": 0.9870821237564087, + "val/ratio_var": 8.347785478690639e-05 + }, + { + "episode": 14176, + "epoch": 0.7159595959595959, + "eps": 0, + "loss/policy_avg": -0.15033547580242157, + "loss/value_avg": 0.6043248772621155, + "lr": 2.0632353839672415e-06, + "objective/entropy": -84.31056213378906, + "objective/kl": 32.41828918457031, + "objective/non_score_reward": -1.6209145784378052, + "objective/rlhf_reward": 15.332210540771484, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008589382632635534, + "policy/clipfrac_avg": 0.002752305706962943, + "policy/entropy_avg": 0.18497180938720703, + "step": 886, + "val/clipfrac_avg": 0.03947104513645172, + "val/num_eos_tokens": 180, + "val/ratio": 0.9879313707351685, + "val/ratio_var": 5.601351585937664e-05 + }, + { + "episode": 14192, + "epoch": 0.7167676767676767, + "eps": 0, + "loss/policy_avg": 0.2279771864414215, + "loss/value_avg": 1.2074341773986816, + "lr": 2.0524355524417017e-06, + "objective/entropy": -51.082984924316406, + "objective/kl": 36.84368133544922, + "objective/non_score_reward": -1.842184066772461, + "objective/rlhf_reward": 14.087503433227539, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010461971396580338, + "policy/clipfrac_avg": 0.004587469156831503, + "policy/entropy_avg": 0.22597980499267578, + "step": 887, + "val/clipfrac_avg": 0.015519710257649422, + "val/num_eos_tokens": 448, + "val/ratio": 0.9849185943603516, + "val/ratio_var": 6.34926327620633e-05 + }, + { + "episode": 14208, + "epoch": 0.7175757575757575, + "eps": 0, + "loss/policy_avg": -0.026267291978001595, + "loss/value_avg": 0.5965927839279175, + "lr": 2.041656756189184e-06, + "objective/entropy": -154.47012329101562, + "objective/kl": 42.720699310302734, + "objective/non_score_reward": -2.1360349655151367, + "objective/rlhf_reward": 14.957715034484863, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009299394441768527, + "policy/clipfrac_avg": 0.008321288973093033, + "policy/entropy_avg": 0.1751893162727356, + "step": 888, + "val/clipfrac_avg": 0.01875000074505806, + "val/num_eos_tokens": 428, + "val/ratio": 0.9883604049682617, + "val/ratio_var": 7.325677870539948e-05 + }, + { + "episode": 14224, + "epoch": 0.7183838383838383, + "eps": 0, + "loss/policy_avg": 0.40078702569007874, + "loss/value_avg": 1.1534854173660278, + "lr": 2.030899072132493e-06, + "objective/entropy": -128.48831176757812, + "objective/kl": 33.898155212402344, + "objective/non_score_reward": -1.6949079036712646, + "objective/rlhf_reward": 12.598060607910156, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006705562118440866, + "policy/clipfrac_avg": 0.005831941030919552, + "policy/entropy_avg": 0.21662330627441406, + "step": 889, + "val/clipfrac_avg": 0.02483724057674408, + "val/num_eos_tokens": 47, + "val/ratio": 0.9912885427474976, + "val/ratio_var": 7.058407936710864e-05 + }, + { + "episode": 14240, + "epoch": 0.7191919191919192, + "eps": 0, + "loss/policy_avg": -0.0855930745601654, + "loss/value_avg": 0.32921552658081055, + "lr": 2.0201625770437647e-06, + "objective/entropy": -86.9806137084961, + "objective/kl": 40.06652069091797, + "objective/non_score_reward": -2.003325939178467, + "objective/rlhf_reward": 14.035737037658691, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008815613109618425, + "policy/clipfrac_avg": 0.006017718929797411, + "policy/entropy_avg": 0.1755352020263672, + "step": 890, + "val/clipfrac_avg": 0.06256753951311111, + "val/num_eos_tokens": 20, + "val/ratio": 0.9872984290122986, + "val/ratio_var": 8.527067984687164e-05 + }, + { + "episode": 14256, + "epoch": 0.72, + "eps": 0, + "loss/policy_avg": -0.17781905829906464, + "loss/value_avg": 0.2275725156068802, + "lr": 2.00944734754392e-06, + "objective/entropy": -99.16207122802734, + "objective/kl": 41.95650863647461, + "objective/non_score_reward": -2.097825288772583, + "objective/rlhf_reward": 13.74592399597168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007897773757576942, + "policy/clipfrac_avg": 0.0043031517416238785, + "policy/entropy_avg": 0.1585559844970703, + "step": 891, + "val/clipfrac_avg": 0.012334126979112625, + "val/num_eos_tokens": 424, + "val/ratio": 0.9894897937774658, + "val/ratio_var": 5.430886085378006e-05 + }, + { + "episode": 14272, + "epoch": 0.7208080808080808, + "eps": 0, + "loss/policy_avg": -0.36396488547325134, + "loss/value_avg": 0.8662234544754028, + "lr": 1.9987534601021225e-06, + "objective/entropy": -61.42802047729492, + "objective/kl": 37.07469177246094, + "objective/non_score_reward": -1.8537347316741943, + "objective/rlhf_reward": 13.044702529907227, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008486385922878981, + "policy/clipfrac_avg": 0.006757801398634911, + "policy/entropy_avg": 0.1866321563720703, + "step": 892, + "val/clipfrac_avg": 0.054801926016807556, + "val/num_eos_tokens": 584, + "val/ratio": 0.9883260726928711, + "val/ratio_var": 6.7303488322068e-05 + }, + { + "episode": 14288, + "epoch": 0.7216161616161616, + "eps": 0, + "loss/policy_avg": -0.12171509861946106, + "loss/value_avg": 0.3499939441680908, + "lr": 1.988080991035223e-06, + "objective/entropy": -58.38874053955078, + "objective/kl": 42.03372573852539, + "objective/non_score_reward": -2.1016862392425537, + "objective/rlhf_reward": 12.828001022338867, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008078413084149361, + "policy/clipfrac_avg": 0.004710954613983631, + "policy/entropy_avg": 0.16219139099121094, + "step": 893, + "val/clipfrac_avg": 0.03385913744568825, + "val/num_eos_tokens": 211, + "val/ratio": 0.9909918904304504, + "val/ratio_var": 4.974031253368594e-05 + }, + { + "episode": 14304, + "epoch": 0.7224242424242424, + "eps": 0, + "loss/policy_avg": 0.15615974366664886, + "loss/value_avg": 0.4985978901386261, + "lr": 1.977430016507222e-06, + "objective/entropy": -72.03929138183594, + "objective/kl": 43.86260986328125, + "objective/non_score_reward": -2.1931304931640625, + "objective/rlhf_reward": 13.322494506835938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009859772399067879, + "policy/clipfrac_avg": 0.004861406981945038, + "policy/entropy_avg": 0.19806480407714844, + "step": 894, + "val/clipfrac_avg": 0.008869262412190437, + "val/num_eos_tokens": 389, + "val/ratio": 0.9869673252105713, + "val/ratio_var": 5.9525649703573436e-05 + }, + { + "episode": 14320, + "epoch": 0.7232323232323232, + "eps": 0, + "loss/policy_avg": -0.018653089180588722, + "loss/value_avg": 0.3236011266708374, + "lr": 1.966800612528723e-06, + "objective/entropy": -30.8804874420166, + "objective/kl": 38.795677185058594, + "objective/non_score_reward": -1.9397838115692139, + "objective/rlhf_reward": 12.958653450012207, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011067797895520926, + "policy/clipfrac_avg": 0.004753993824124336, + "policy/entropy_avg": 0.21401214599609375, + "step": 895, + "val/clipfrac_avg": 0.013285137712955475, + "val/num_eos_tokens": 190, + "val/ratio": 0.9844212532043457, + "val/ratio_var": 5.578770287684165e-05 + }, + { + "episode": 14336, + "epoch": 0.724040404040404, + "eps": 0, + "loss/policy_avg": -0.04005177319049835, + "loss/value_avg": 0.564488410949707, + "lr": 1.956192854956397e-06, + "objective/entropy": -88.01570129394531, + "objective/kl": 40.9593505859375, + "objective/non_score_reward": -2.0479674339294434, + "objective/rlhf_reward": 13.350469589233398, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010309319477528334, + "policy/clipfrac_avg": 0.005483411252498627, + "policy/entropy_avg": 0.21037864685058594, + "step": 896, + "val/clipfrac_avg": 0.01046415138989687, + "val/num_eos_tokens": 539, + "val/ratio": 0.9865807890892029, + "val/ratio_var": 9.776368096936494e-05 + }, + { + "episode": 14352, + "epoch": 0.7248484848484849, + "eps": 0, + "loss/policy_avg": -0.1358407437801361, + "loss/value_avg": 0.536889910697937, + "lr": 1.945606819492429e-06, + "objective/entropy": -98.2094955444336, + "objective/kl": 37.47423553466797, + "objective/non_score_reward": -1.8737118244171143, + "objective/rlhf_reward": 13.634100914001465, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007985340780578554, + "policy/clipfrac_avg": 0.005063176155090332, + "policy/entropy_avg": 0.14582359790802002, + "step": 897, + "val/clipfrac_avg": 0.06062069162726402, + "val/num_eos_tokens": 1182, + "val/ratio": 0.9903557300567627, + "val/ratio_var": 5.06751166540198e-05 + }, + { + "episode": 14368, + "epoch": 0.7256565656565657, + "eps": 0, + "loss/policy_avg": 0.011427253484725952, + "loss/value_avg": 0.263162761926651, + "lr": 1.9350425816839875e-06, + "objective/entropy": -110.30941772460938, + "objective/kl": 41.13695526123047, + "objective/non_score_reward": -2.0568478107452393, + "objective/rlhf_reward": 13.72440242767334, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011626393534243107, + "policy/clipfrac_avg": 0.006754214409738779, + "policy/entropy_avg": 0.2226581573486328, + "step": 898, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 10, + "val/ratio": 0.9850178956985474, + "val/ratio_var": 8.876781794242561e-05 + }, + { + "episode": 14384, + "epoch": 0.7264646464646465, + "eps": 0, + "loss/policy_avg": 0.11541193723678589, + "loss/value_avg": 0.4631091356277466, + "lr": 1.9245002169226814e-06, + "objective/entropy": -101.70489501953125, + "objective/kl": 48.557960510253906, + "objective/non_score_reward": -2.4278979301452637, + "objective/rlhf_reward": 13.314289093017578, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009826980531215668, + "policy/clipfrac_avg": 0.004610124975442886, + "policy/entropy_avg": 0.1724681854248047, + "step": 899, + "val/clipfrac_avg": 0.013620924204587936, + "val/num_eos_tokens": 516, + "val/ratio": 0.9876973628997803, + "val/ratio_var": 5.4934793297434226e-05 + }, + { + "episode": 14400, + "epoch": 0.7272727272727273, + "eps": 0, + "loss/policy_avg": 0.07936800271272659, + "loss/value_avg": 0.6616516709327698, + "lr": 1.913979800444021e-06, + "objective/entropy": -91.96019744873047, + "objective/kl": 41.26939010620117, + "objective/non_score_reward": -2.063469409942627, + "objective/rlhf_reward": 12.522467613220215, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0009332017507404089, + "policy/clipfrac_avg": 0.0043076397851109505, + "policy/entropy_avg": 0.18123245239257812, + "step": 900, + "val/clipfrac_avg": 0.01476163137704134, + "val/num_eos_tokens": 595, + "val/ratio": 0.9875973463058472, + "val/ratio_var": 7.475823076674715e-05 + }, + { + "episode": 14416, + "epoch": 0.7280808080808081, + "eps": 0, + "loss/policy_avg": -0.02671997621655464, + "loss/value_avg": 0.7174835205078125, + "lr": 1.9034814073268898e-06, + "objective/entropy": -59.236602783203125, + "objective/kl": 39.73054122924805, + "objective/non_score_reward": -1.9865270853042603, + "objective/rlhf_reward": 12.435348510742188, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011704354546964169, + "policy/clipfrac_avg": 0.004937891382724047, + "policy/entropy_avg": 0.22887516021728516, + "step": 901, + "val/clipfrac_avg": 0.13261640071868896, + "val/num_eos_tokens": 121, + "val/ratio": 0.9848372936248779, + "val/ratio_var": 8.883672853698954e-05 + }, + { + "episode": 14432, + "epoch": 0.7288888888888889, + "eps": 0, + "loss/policy_avg": 0.05788304656744003, + "loss/value_avg": 0.2591496407985687, + "lr": 1.8930051124929904e-06, + "objective/entropy": -53.46650695800781, + "objective/kl": 46.268035888671875, + "objective/non_score_reward": -2.313401937484741, + "objective/rlhf_reward": 14.38972282409668, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010022292844951153, + "policy/clipfrac_avg": 0.004066841676831245, + "policy/entropy_avg": 0.20316123962402344, + "step": 902, + "val/clipfrac_avg": 0.013965151272714138, + "val/num_eos_tokens": 455, + "val/ratio": 0.9866280555725098, + "val/ratio_var": 5.229823727859184e-05 + }, + { + "episode": 14448, + "epoch": 0.7296969696969697, + "eps": 0, + "loss/policy_avg": -0.11720985174179077, + "loss/value_avg": 0.23323997855186462, + "lr": 1.8825509907063328e-06, + "objective/entropy": -65.6883773803711, + "objective/kl": 46.209991455078125, + "objective/non_score_reward": -2.310499668121338, + "objective/rlhf_reward": 16.78325080871582, + "objective/scores": 19.125, + "policy/approxkl_avg": 0.0010066109243780375, + "policy/clipfrac_avg": 0.005406823009252548, + "policy/entropy_avg": 0.20101547241210938, + "step": 903, + "val/clipfrac_avg": 0.015051587484776974, + "val/num_eos_tokens": 205, + "val/ratio": 0.9865299463272095, + "val/ratio_var": 5.01565846207086e-05 + }, + { + "episode": 14464, + "epoch": 0.7305050505050505, + "eps": 0, + "loss/policy_avg": -0.4681462049484253, + "loss/value_avg": 0.8052223920822144, + "lr": 1.87211911657268e-06, + "objective/entropy": -118.0733642578125, + "objective/kl": 37.29925537109375, + "objective/non_score_reward": -1.8649628162384033, + "objective/rlhf_reward": 15.885037422180176, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0007588897133246064, + "policy/clipfrac_avg": 0.0023906652349978685, + "policy/entropy_avg": 0.15973949432373047, + "step": 904, + "val/clipfrac_avg": 0.13810205459594727, + "val/num_eos_tokens": 155, + "val/ratio": 0.9902135729789734, + "val/ratio_var": 7.25651771062985e-05 + }, + { + "episode": 14480, + "epoch": 0.7313131313131314, + "eps": 0, + "loss/policy_avg": 0.19160020351409912, + "loss/value_avg": 0.9071890115737915, + "lr": 1.8617095645390332e-06, + "objective/entropy": -22.496440887451172, + "objective/kl": 39.160484313964844, + "objective/non_score_reward": -1.958024263381958, + "objective/rlhf_reward": 12.323225975036621, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0010718535631895065, + "policy/clipfrac_avg": 0.004276500083506107, + "policy/entropy_avg": 0.19223785400390625, + "step": 905, + "val/clipfrac_avg": 0.03053325042128563, + "val/num_eos_tokens": 240, + "val/ratio": 0.9871090650558472, + "val/ratio_var": 4.5721961214439943e-05 + }, + { + "episode": 14496, + "epoch": 0.7321212121212122, + "eps": 0, + "loss/policy_avg": -0.11877916008234024, + "loss/value_avg": 0.6873422265052795, + "lr": 1.8513224088930814e-06, + "objective/entropy": -52.41043472290039, + "objective/kl": 30.631162643432617, + "objective/non_score_reward": -1.5315582752227783, + "objective/rlhf_reward": 16.296566009521484, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.001107763615436852, + "policy/clipfrac_avg": 0.006524171680212021, + "policy/entropy_avg": 0.2064342498779297, + "step": 906, + "val/clipfrac_avg": 0.000294811325147748, + "val/num_eos_tokens": 261, + "val/ratio": 0.9861389398574829, + "val/ratio_var": 6.195842433953658e-05 + }, + { + "episode": 14512, + "epoch": 0.7329292929292929, + "eps": 0, + "loss/policy_avg": 0.10565789043903351, + "loss/value_avg": 0.5067005157470703, + "lr": 1.8409577237626935e-06, + "objective/entropy": -61.856544494628906, + "objective/kl": 43.33234405517578, + "objective/non_score_reward": -2.1666173934936523, + "objective/rlhf_reward": 13.145882606506348, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010527942795306444, + "policy/clipfrac_avg": 0.004575061611831188, + "policy/entropy_avg": 0.18899154663085938, + "step": 907, + "val/clipfrac_avg": 0.005580357275903225, + "val/num_eos_tokens": 496, + "val/ratio": 0.9872788190841675, + "val/ratio_var": 7.066375110298395e-05 + }, + { + "episode": 14528, + "epoch": 0.7337373737373737, + "eps": 0, + "loss/policy_avg": 0.04191252216696739, + "loss/value_avg": 0.30707108974456787, + "lr": 1.8306155831153699e-06, + "objective/entropy": -91.67915344238281, + "objective/kl": 37.619895935058594, + "objective/non_score_reward": -1.8809947967529297, + "objective/rlhf_reward": 14.22056770324707, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009494669502601027, + "policy/clipfrac_avg": 0.007099117152392864, + "policy/entropy_avg": 0.17319107055664062, + "step": 908, + "val/clipfrac_avg": 0.07242873311042786, + "val/num_eos_tokens": 437, + "val/ratio": 0.9883796572685242, + "val/ratio_var": 7.36714864615351e-05 + }, + { + "episode": 14544, + "epoch": 0.7345454545454545, + "eps": 0, + "loss/policy_avg": 0.15317559242248535, + "loss/value_avg": 0.7379442453384399, + "lr": 1.8202960607577246e-06, + "objective/entropy": -92.03844451904297, + "objective/kl": 36.91444396972656, + "objective/non_score_reward": -1.8457221984863281, + "objective/rlhf_reward": 13.802715301513672, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0009770962642505765, + "policy/clipfrac_avg": 0.005569060333073139, + "policy/entropy_avg": 0.18958091735839844, + "step": 909, + "val/clipfrac_avg": 0.04051949828863144, + "val/num_eos_tokens": 56, + "val/ratio": 0.988923192024231, + "val/ratio_var": 6.142368511063978e-05 + }, + { + "episode": 14560, + "epoch": 0.7353535353535353, + "eps": 0, + "loss/policy_avg": 0.1602431833744049, + "loss/value_avg": 0.3025469183921814, + "lr": 1.809999230334958e-06, + "objective/entropy": -111.80148315429688, + "objective/kl": 38.83802032470703, + "objective/non_score_reward": -1.9419012069702148, + "objective/rlhf_reward": 14.401848793029785, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000904000480659306, + "policy/clipfrac_avg": 0.005316127557307482, + "policy/entropy_avg": 0.17273902893066406, + "step": 910, + "val/clipfrac_avg": 0.00018382353300694376, + "val/num_eos_tokens": 342, + "val/ratio": 0.9891130924224854, + "val/ratio_var": 7.234779332065955e-05 + }, + { + "episode": 14576, + "epoch": 0.7361616161616161, + "eps": 0, + "loss/policy_avg": 0.09882547706365585, + "loss/value_avg": 0.6121327877044678, + "lr": 1.7997251653303249e-06, + "objective/entropy": -98.02986145019531, + "objective/kl": 44.55670166015625, + "objective/non_score_reward": -2.227834939956665, + "objective/rlhf_reward": 13.092477798461914, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009336632210761309, + "policy/clipfrac_avg": 0.004822542425245047, + "policy/entropy_avg": 0.17920780181884766, + "step": 911, + "val/clipfrac_avg": 0.0050772735849022865, + "val/num_eos_tokens": 426, + "val/ratio": 0.9875223636627197, + "val/ratio_var": 6.380004924722016e-05 + }, + { + "episode": 14592, + "epoch": 0.7369696969696969, + "eps": 0, + "loss/policy_avg": -0.15951846539974213, + "loss/value_avg": 0.5648845434188843, + "lr": 1.7894739390646227e-06, + "objective/entropy": -57.16059875488281, + "objective/kl": 36.05466079711914, + "objective/non_score_reward": -1.802733063697815, + "objective/rlhf_reward": 13.119142532348633, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009670511353760958, + "policy/clipfrac_avg": 0.00586615689098835, + "policy/entropy_avg": 0.1721019744873047, + "step": 912, + "val/clipfrac_avg": 0.0008223684271797538, + "val/num_eos_tokens": 45, + "val/ratio": 0.9892112016677856, + "val/ratio_var": 4.9591733841225505e-05 + }, + { + "episode": 14608, + "epoch": 0.7377777777777778, + "eps": 0, + "loss/policy_avg": 0.1940194070339203, + "loss/value_avg": 0.3815060555934906, + "lr": 1.7792456246956535e-06, + "objective/entropy": -146.6658935546875, + "objective/kl": 46.18011474609375, + "objective/non_score_reward": -2.3090057373046875, + "objective/rlhf_reward": 14.511306762695312, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009307245491072536, + "policy/clipfrac_avg": 0.006143895909190178, + "policy/entropy_avg": 0.18556424975395203, + "step": 913, + "val/clipfrac_avg": 0.0180903859436512, + "val/num_eos_tokens": 698, + "val/ratio": 0.9874189496040344, + "val/ratio_var": 0.00011522904242156073 + }, + { + "episode": 14624, + "epoch": 0.7385858585858586, + "eps": 0, + "loss/policy_avg": -0.02095976285636425, + "loss/value_avg": 0.7224372625350952, + "lr": 1.7690402952177117e-06, + "objective/entropy": -84.84890747070312, + "objective/kl": 35.78252410888672, + "objective/non_score_reward": -1.7891263961791992, + "objective/rlhf_reward": 13.2655611038208, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009321504039689898, + "policy/clipfrac_avg": 0.005007730796933174, + "policy/entropy_avg": 0.18702125549316406, + "step": 914, + "val/clipfrac_avg": 0.023768942803144455, + "val/num_eos_tokens": 403, + "val/ratio": 0.9874284267425537, + "val/ratio_var": 8.100390550680459e-05 + }, + { + "episode": 14640, + "epoch": 0.7393939393939394, + "eps": 0, + "loss/policy_avg": -0.3134731650352478, + "loss/value_avg": 0.5765621066093445, + "lr": 1.7588580234610592e-06, + "objective/entropy": -63.22113037109375, + "objective/kl": 38.5015869140625, + "objective/non_score_reward": -1.925079345703125, + "objective/rlhf_reward": 13.918670654296875, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008501409320160747, + "policy/clipfrac_avg": 0.004706401377916336, + "policy/entropy_avg": 0.1600513458251953, + "step": 915, + "val/clipfrac_avg": 0.06931675970554352, + "val/num_eos_tokens": 171, + "val/ratio": 0.9882992506027222, + "val/ratio_var": 6.202542135724798e-05 + }, + { + "episode": 14656, + "epoch": 0.7402020202020202, + "eps": 0, + "loss/policy_avg": -0.2850199043750763, + "loss/value_avg": 0.8679437637329102, + "lr": 1.7486988820914063e-06, + "objective/entropy": -43.271400451660156, + "objective/kl": 42.955291748046875, + "objective/non_score_reward": -2.1477646827697754, + "objective/rlhf_reward": 13.875673294067383, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009495788253843784, + "policy/clipfrac_avg": 0.005583056248724461, + "policy/entropy_avg": 0.17831039428710938, + "step": 916, + "val/clipfrac_avg": 0.06379853188991547, + "val/num_eos_tokens": 103, + "val/ratio": 0.9876798391342163, + "val/ratio_var": 4.0028760849963874e-05 + }, + { + "episode": 14672, + "epoch": 0.741010101010101, + "eps": 0, + "loss/policy_avg": 0.21622401475906372, + "loss/value_avg": 0.22584834694862366, + "lr": 1.7385629436093958e-06, + "objective/entropy": -79.67193603515625, + "objective/kl": 44.639923095703125, + "objective/non_score_reward": -2.2319962978363037, + "objective/rlhf_reward": 13.463315963745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008941494161263108, + "policy/clipfrac_avg": 0.006674608215689659, + "policy/entropy_avg": 0.1719951629638672, + "step": 917, + "val/clipfrac_avg": 0.029584797099232674, + "val/num_eos_tokens": 50, + "val/ratio": 0.9886019825935364, + "val/ratio_var": 4.6610963181592524e-05 + }, + { + "episode": 14688, + "epoch": 0.7418181818181818, + "eps": 0, + "loss/policy_avg": -0.0006486065685749054, + "loss/value_avg": 0.2962718605995178, + "lr": 1.728450280350079e-06, + "objective/entropy": -68.00552368164062, + "objective/kl": 38.6593132019043, + "objective/non_score_reward": -1.9329655170440674, + "objective/rlhf_reward": 13.887346267700195, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008251155959442258, + "policy/clipfrac_avg": 0.005509347654879093, + "policy/entropy_avg": 0.1643085479736328, + "step": 918, + "val/clipfrac_avg": 0.003826581174507737, + "val/num_eos_tokens": 44, + "val/ratio": 0.9888970851898193, + "val/ratio_var": 3.0508910640492104e-05 + }, + { + "episode": 14704, + "epoch": 0.7426262626262626, + "eps": 0, + "loss/policy_avg": -0.3462671935558319, + "loss/value_avg": 0.7152901887893677, + "lr": 1.7183609644824096e-06, + "objective/entropy": -93.26936340332031, + "objective/kl": 40.42371368408203, + "objective/non_score_reward": -2.021185874938965, + "objective/rlhf_reward": 15.166314125061035, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009743100963532925, + "policy/clipfrac_avg": 0.0028937258757650852, + "policy/entropy_avg": 0.18361568450927734, + "step": 919, + "val/clipfrac_avg": 0.049027521163225174, + "val/num_eos_tokens": 136, + "val/ratio": 0.9877507090568542, + "val/ratio_var": 5.374600732466206e-05 + }, + { + "episode": 14720, + "epoch": 0.7434343434343434, + "eps": 0, + "loss/policy_avg": 0.1274300217628479, + "loss/value_avg": 1.230905294418335, + "lr": 1.708295068008718e-06, + "objective/entropy": -81.48149108886719, + "objective/kl": 36.17670440673828, + "objective/non_score_reward": -1.8088352680206299, + "objective/rlhf_reward": 12.589601516723633, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008495738729834557, + "policy/clipfrac_avg": 0.0038070818409323692, + "policy/entropy_avg": 0.16942787170410156, + "step": 920, + "val/clipfrac_avg": 0.007427365053445101, + "val/num_eos_tokens": 270, + "val/ratio": 0.9902092218399048, + "val/ratio_var": 5.2669711294583976e-05 + }, + { + "episode": 14736, + "epoch": 0.7442424242424243, + "eps": 0, + "loss/policy_avg": -0.257866770029068, + "loss/value_avg": 0.5942734479904175, + "lr": 1.6982526627642043e-06, + "objective/entropy": -74.04690551757812, + "objective/kl": 42.419307708740234, + "objective/non_score_reward": -2.1209654808044434, + "objective/rlhf_reward": 13.511846542358398, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000907333567738533, + "policy/clipfrac_avg": 0.004857117310166359, + "policy/entropy_avg": 0.18361282348632812, + "step": 921, + "val/clipfrac_avg": 0.043217435479164124, + "val/num_eos_tokens": 56, + "val/ratio": 0.9877738356590271, + "val/ratio_var": 6.433074304368347e-05 + }, + { + "episode": 14752, + "epoch": 0.7450505050505051, + "eps": 0, + "loss/policy_avg": 0.12189102917909622, + "loss/value_avg": 0.41360408067703247, + "lr": 1.6882338204164212e-06, + "objective/entropy": -54.975791931152344, + "objective/kl": 35.72700881958008, + "objective/non_score_reward": -1.7863504886627197, + "objective/rlhf_reward": 13.39333724975586, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010183433769270778, + "policy/clipfrac_avg": 0.003768398892134428, + "policy/entropy_avg": 0.1844940185546875, + "step": 922, + "val/clipfrac_avg": 0.0008610248914919794, + "val/num_eos_tokens": 362, + "val/ratio": 0.9872738122940063, + "val/ratio_var": 3.0919560231268406e-05 + }, + { + "episode": 14768, + "epoch": 0.7458585858585859, + "eps": 0, + "loss/policy_avg": -0.07059650868177414, + "loss/value_avg": 0.6463966369628906, + "lr": 1.67823861246477e-06, + "objective/entropy": -15.746962547302246, + "objective/kl": 40.59729766845703, + "objective/non_score_reward": -2.029865026473999, + "objective/rlhf_reward": 12.388103485107422, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010770086664706469, + "policy/clipfrac_avg": 0.005413630045950413, + "policy/entropy_avg": 0.21207809448242188, + "step": 923, + "val/clipfrac_avg": 0.00856552179902792, + "val/num_eos_tokens": 175, + "val/ratio": 0.9861268401145935, + "val/ratio_var": 5.63512985536363e-05 + }, + { + "episode": 14784, + "epoch": 0.7466666666666667, + "eps": 0, + "loss/policy_avg": 0.06915351748466492, + "loss/value_avg": 1.4651566743850708, + "lr": 1.6682671102399806e-06, + "objective/entropy": -118.91422271728516, + "objective/kl": 34.25519943237305, + "objective/non_score_reward": -1.7127599716186523, + "objective/rlhf_reward": 12.224739074707031, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0007098158821463585, + "policy/clipfrac_avg": 0.0022262982092797756, + "policy/entropy_avg": 0.1386256217956543, + "step": 924, + "val/clipfrac_avg": 0.06050518900156021, + "val/num_eos_tokens": 270, + "val/ratio": 0.9912421703338623, + "val/ratio_var": 4.530951628112234e-05 + }, + { + "episode": 14800, + "epoch": 0.7474747474747475, + "eps": 0, + "loss/policy_avg": 0.18248242139816284, + "loss/value_avg": 1.6025516986846924, + "lr": 1.6583193849036095e-06, + "objective/entropy": -81.5009765625, + "objective/kl": 38.95854949951172, + "objective/non_score_reward": -1.947927474975586, + "objective/rlhf_reward": 15.450510025024414, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009277502540498972, + "policy/clipfrac_avg": 0.004952313844114542, + "policy/entropy_avg": 0.16534215211868286, + "step": 925, + "val/clipfrac_avg": 0.016582708805799484, + "val/num_eos_tokens": 323, + "val/ratio": 0.9883300065994263, + "val/ratio_var": 5.890153624932282e-05 + }, + { + "episode": 14816, + "epoch": 0.7482828282828283, + "eps": 0, + "loss/policy_avg": -0.16795751452445984, + "loss/value_avg": 0.3992899954319, + "lr": 1.6483955074475278e-06, + "objective/entropy": -46.540870666503906, + "objective/kl": 35.18474578857422, + "objective/non_score_reward": -1.75923752784729, + "objective/rlhf_reward": 13.225137710571289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009727986180223525, + "policy/clipfrac_avg": 0.0023795529268682003, + "policy/entropy_avg": 0.21690034866333008, + "step": 926, + "val/clipfrac_avg": 0.0444885715842247, + "val/num_eos_tokens": 163, + "val/ratio": 0.9869061708450317, + "val/ratio_var": 6.481046148110181e-05 + }, + { + "episode": 14832, + "epoch": 0.7490909090909091, + "eps": 0, + "loss/policy_avg": -0.3186694383621216, + "loss/value_avg": 0.933090090751648, + "lr": 1.6384955486934157e-06, + "objective/entropy": -73.27378845214844, + "objective/kl": 33.047142028808594, + "objective/non_score_reward": -1.6523573398590088, + "objective/rlhf_reward": 13.33201789855957, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008205980993807316, + "policy/clipfrac_avg": 0.0039194016717374325, + "policy/entropy_avg": 0.17404580116271973, + "step": 927, + "val/clipfrac_avg": 0.07463380694389343, + "val/num_eos_tokens": 165, + "val/ratio": 0.9890973567962646, + "val/ratio_var": 8.332987636094913e-05 + }, + { + "episode": 14848, + "epoch": 0.74989898989899, + "eps": 0, + "loss/policy_avg": -0.018651030957698822, + "loss/value_avg": 0.23730480670928955, + "lr": 1.6286195792922655e-06, + "objective/entropy": -108.95191192626953, + "objective/kl": 41.2000846862793, + "objective/non_score_reward": -2.060004472732544, + "objective/rlhf_reward": 14.135308265686035, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007390918326564133, + "policy/clipfrac_avg": 0.0021800659596920013, + "policy/entropy_avg": 0.15365886688232422, + "step": 928, + "val/clipfrac_avg": 0.02866453304886818, + "val/num_eos_tokens": 497, + "val/ratio": 0.989920973777771, + "val/ratio_var": 5.118203625897877e-05 + }, + { + "episode": 14864, + "epoch": 0.7507070707070707, + "eps": 0, + "loss/policy_avg": 0.11356082558631897, + "loss/value_avg": 0.5069226622581482, + "lr": 1.6187676697238564e-06, + "objective/entropy": -53.78815841674805, + "objective/kl": 47.88815689086914, + "objective/non_score_reward": -2.3944079875946045, + "objective/rlhf_reward": 12.574342727661133, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008938192040659487, + "policy/clipfrac_avg": 0.003149396274238825, + "policy/entropy_avg": 0.181840181350708, + "step": 929, + "val/clipfrac_avg": 0.0172991082072258, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889012575149536, + "val/ratio_var": 3.912192551069893e-05 + }, + { + "episode": 14880, + "epoch": 0.7515151515151515, + "eps": 0, + "loss/policy_avg": -0.07865975797176361, + "loss/value_avg": 0.41433531045913696, + "lr": 1.6089398902962767e-06, + "objective/entropy": -104.1807861328125, + "objective/kl": 37.49846649169922, + "objective/non_score_reward": -1.8749234676361084, + "objective/rlhf_reward": 15.953201293945312, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009115774882957339, + "policy/clipfrac_avg": 0.005451949778944254, + "policy/entropy_avg": 0.1745290756225586, + "step": 930, + "val/clipfrac_avg": 0.058040205389261246, + "val/num_eos_tokens": 27, + "val/ratio": 0.9887331128120422, + "val/ratio_var": 6.705939449602738e-05 + }, + { + "episode": 14896, + "epoch": 0.7523232323232323, + "eps": 0, + "loss/policy_avg": -0.035402484238147736, + "loss/value_avg": 0.5358971357345581, + "lr": 1.5991363111454023e-06, + "objective/entropy": -97.50758361816406, + "objective/kl": 42.68102264404297, + "objective/non_score_reward": -2.1340510845184326, + "objective/rlhf_reward": 13.498762130737305, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008075715741142631, + "policy/clipfrac_avg": 0.011311197653412819, + "policy/entropy_avg": 0.1630105972290039, + "step": 931, + "val/clipfrac_avg": 0.003885046113282442, + "val/num_eos_tokens": 302, + "val/ratio": 0.9890309572219849, + "val/ratio_var": 5.2104482165304944e-05 + }, + { + "episode": 14912, + "epoch": 0.7531313131313131, + "eps": 0, + "loss/policy_avg": -0.30045461654663086, + "loss/value_avg": 0.872713565826416, + "lr": 1.5893570022344124e-06, + "objective/entropy": -78.21574401855469, + "objective/kl": 32.467742919921875, + "objective/non_score_reward": -1.623387098312378, + "objective/rlhf_reward": 14.372706413269043, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008647768991068006, + "policy/clipfrac_avg": 0.005656212102621794, + "policy/entropy_avg": 0.17721939086914062, + "step": 932, + "val/clipfrac_avg": 0.0015919646248221397, + "val/num_eos_tokens": 12, + "val/ratio": 0.9885766506195068, + "val/ratio_var": 6.424664752557874e-05 + }, + { + "episode": 14928, + "epoch": 0.7539393939393939, + "eps": 0, + "loss/policy_avg": -0.012097490951418877, + "loss/value_avg": 0.7004050016403198, + "lr": 1.5796020333532696e-06, + "objective/entropy": -61.095802307128906, + "objective/kl": 36.504150390625, + "objective/non_score_reward": -1.8252074718475342, + "objective/rlhf_reward": 12.393542289733887, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008130621863529086, + "policy/clipfrac_avg": 0.005108194425702095, + "policy/entropy_avg": 0.16521072387695312, + "step": 933, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 182, + "val/ratio": 0.9887269735336304, + "val/ratio_var": 6.136602314654738e-05 + }, + { + "episode": 14944, + "epoch": 0.7547474747474747, + "eps": 0, + "loss/policy_avg": 0.03149443119764328, + "loss/value_avg": 0.3308320641517639, + "lr": 1.5698714741182459e-06, + "objective/entropy": -110.07270812988281, + "objective/kl": 34.973388671875, + "objective/non_score_reward": -1.7486693859100342, + "objective/rlhf_reward": 14.548205375671387, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00091616774443537, + "policy/clipfrac_avg": 0.006772395223379135, + "policy/entropy_avg": 0.17972993850708008, + "step": 934, + "val/clipfrac_avg": 0.0015980113530531526, + "val/num_eos_tokens": 350, + "val/ratio": 0.9878275394439697, + "val/ratio_var": 8.512718341080472e-05 + }, + { + "episode": 14960, + "epoch": 0.7555555555555555, + "eps": 0, + "loss/policy_avg": -0.03353989124298096, + "loss/value_avg": 0.4103987216949463, + "lr": 1.5601653939714073e-06, + "objective/entropy": -25.034250259399414, + "objective/kl": 39.53004837036133, + "objective/non_score_reward": -1.976502537727356, + "objective/rlhf_reward": 14.281310081481934, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102006527595222, + "policy/clipfrac_avg": 0.00497299712151289, + "policy/entropy_avg": 0.19387245178222656, + "step": 935, + "val/clipfrac_avg": 0.0242628026753664, + "val/num_eos_tokens": 73, + "val/ratio": 0.9875220060348511, + "val/ratio_var": 6.656184268649668e-05 + }, + { + "episode": 14976, + "epoch": 0.7563636363636363, + "eps": 0, + "loss/policy_avg": -0.1474141776561737, + "loss/value_avg": 0.527273416519165, + "lr": 1.5504838621801272e-06, + "objective/entropy": -60.77607345581055, + "objective/kl": 37.98503494262695, + "objective/non_score_reward": -1.8992516994476318, + "objective/rlhf_reward": 13.171060562133789, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007948282873257995, + "policy/clipfrac_avg": 0.002496684668585658, + "policy/entropy_avg": 0.1510624885559082, + "step": 936, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 3.4012387914117426e-05 + }, + { + "episode": 14992, + "epoch": 0.7571717171717172, + "eps": 0, + "loss/policy_avg": 0.1368754655122757, + "loss/value_avg": 0.6238852143287659, + "lr": 1.5408269478365883e-06, + "objective/entropy": -60.082557678222656, + "objective/kl": 41.95027160644531, + "objective/non_score_reward": -2.0975136756896973, + "objective/rlhf_reward": 12.652486801147461, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009200765052810311, + "policy/clipfrac_avg": 0.0059701185673475266, + "policy/entropy_avg": 0.17743873596191406, + "step": 937, + "val/clipfrac_avg": 0.012415202334523201, + "val/num_eos_tokens": 54, + "val/ratio": 0.9879815578460693, + "val/ratio_var": 4.387104490888305e-05 + }, + { + "episode": 15008, + "epoch": 0.757979797979798, + "eps": 0, + "loss/policy_avg": -0.17327642440795898, + "loss/value_avg": 0.538246750831604, + "lr": 1.5311947198572918e-06, + "objective/entropy": -112.668701171875, + "objective/kl": 35.57097625732422, + "objective/non_score_reward": -1.7785489559173584, + "objective/rlhf_reward": 13.416763305664062, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008795931353233755, + "policy/clipfrac_avg": 0.005144854076206684, + "policy/entropy_avg": 0.16583538055419922, + "step": 938, + "val/clipfrac_avg": 0.047972679138183594, + "val/num_eos_tokens": 136, + "val/ratio": 0.9902178645133972, + "val/ratio_var": 5.2144809160381556e-05 + }, + { + "episode": 15024, + "epoch": 0.7587878787878788, + "eps": 0, + "loss/policy_avg": 0.5920745134353638, + "loss/value_avg": 1.1476694345474243, + "lr": 1.5215872469825682e-06, + "objective/entropy": -72.79998779296875, + "objective/kl": 35.88062286376953, + "objective/non_score_reward": -1.7940311431884766, + "objective/rlhf_reward": 14.46378231048584, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008520716801285744, + "policy/clipfrac_avg": 0.0030313697643578053, + "policy/entropy_avg": 0.19948351383209229, + "step": 939, + "val/clipfrac_avg": 0.021162573248147964, + "val/num_eos_tokens": 210, + "val/ratio": 0.9882235527038574, + "val/ratio_var": 6.201990618137643e-05 + }, + { + "episode": 15040, + "epoch": 0.7595959595959596, + "eps": 0, + "loss/policy_avg": -0.15639176964759827, + "loss/value_avg": 0.8813156485557556, + "lr": 1.5120045977760784e-06, + "objective/entropy": -91.86679077148438, + "objective/kl": 42.35734176635742, + "objective/non_score_reward": -2.1178669929504395, + "objective/rlhf_reward": 15.264945983886719, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0010741858277469873, + "policy/clipfrac_avg": 0.00790454912930727, + "policy/entropy_avg": 0.19293498992919922, + "step": 940, + "val/clipfrac_avg": 0.015625, + "val/num_eos_tokens": 9, + "val/ratio": 0.986987829208374, + "val/ratio_var": 6.426944310078397e-05 + }, + { + "episode": 15056, + "epoch": 0.7604040404040404, + "eps": 0, + "loss/policy_avg": 0.2165047824382782, + "loss/value_avg": 0.34554335474967957, + "lr": 1.502446840624331e-06, + "objective/entropy": -62.31670379638672, + "objective/kl": 31.252155303955078, + "objective/non_score_reward": -1.562608003616333, + "objective/rlhf_reward": 14.156142234802246, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007184218848124146, + "policy/clipfrac_avg": 0.0035393601283431053, + "policy/entropy_avg": 0.14359378814697266, + "step": 941, + "val/clipfrac_avg": 0.013209432363510132, + "val/num_eos_tokens": 36, + "val/ratio": 0.9896091222763062, + "val/ratio_var": 5.599396536126733e-05 + }, + { + "episode": 15072, + "epoch": 0.7612121212121212, + "eps": 0, + "loss/policy_avg": -0.030075984075665474, + "loss/value_avg": 0.45760178565979004, + "lr": 1.4929140437361916e-06, + "objective/entropy": -109.5699691772461, + "objective/kl": 38.075103759765625, + "objective/non_score_reward": -1.9037551879882812, + "objective/rlhf_reward": 14.713432312011719, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010412151459604502, + "policy/clipfrac_avg": 0.009930627420544624, + "policy/entropy_avg": 0.18667316436767578, + "step": 942, + "val/clipfrac_avg": 0.014139890670776367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9863598346710205, + "val/ratio_var": 0.00010634912905516103 + }, + { + "episode": 15088, + "epoch": 0.762020202020202, + "eps": 0, + "loss/policy_avg": 0.12722979485988617, + "loss/value_avg": 0.5696362853050232, + "lr": 1.4834062751424018e-06, + "objective/entropy": -121.374267578125, + "objective/kl": 34.23678207397461, + "objective/non_score_reward": -1.711839199066162, + "objective/rlhf_reward": 14.038161277770996, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0007155380444601178, + "policy/clipfrac_avg": 0.00446352269500494, + "policy/entropy_avg": 0.13922500610351562, + "step": 943, + "val/clipfrac_avg": 0.001972501166164875, + "val/num_eos_tokens": 493, + "val/ratio": 0.9906710386276245, + "val/ratio_var": 6.088761438149959e-05 + }, + { + "episode": 15104, + "epoch": 0.7628282828282829, + "eps": 0, + "loss/policy_avg": 0.24459479749202728, + "loss/value_avg": 0.997427225112915, + "lr": 1.4739236026950838e-06, + "objective/entropy": -117.67542266845703, + "objective/kl": 42.60675811767578, + "objective/non_score_reward": -2.130337953567505, + "objective/rlhf_reward": 13.854036331176758, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007362811593338847, + "policy/clipfrac_avg": 0.009282913990318775, + "policy/entropy_avg": 0.15163874626159668, + "step": 944, + "val/clipfrac_avg": 0.019384628161787987, + "val/num_eos_tokens": 404, + "val/ratio": 0.9907106161117554, + "val/ratio_var": 5.1745548262260854e-05 + }, + { + "episode": 15120, + "epoch": 0.7636363636363637, + "eps": 0, + "loss/policy_avg": 0.019024059176445007, + "loss/value_avg": 0.2505372166633606, + "lr": 1.4644660940672628e-06, + "objective/entropy": -72.49942779541016, + "objective/kl": 36.86358642578125, + "objective/non_score_reward": -1.8431792259216309, + "objective/rlhf_reward": 14.227133750915527, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009655736503191292, + "policy/clipfrac_avg": 0.0035610971972346306, + "policy/entropy_avg": 0.19275665283203125, + "step": 945, + "val/clipfrac_avg": 0.0008617835119366646, + "val/num_eos_tokens": 33, + "val/ratio": 0.9873601198196411, + "val/ratio_var": 7.99532572273165e-05 + }, + { + "episode": 15136, + "epoch": 0.7644444444444445, + "eps": 0, + "loss/policy_avg": -0.24745053052902222, + "loss/value_avg": 0.4052670896053314, + "lr": 1.455033816752383e-06, + "objective/entropy": -106.38670349121094, + "objective/kl": 30.419466018676758, + "objective/non_score_reward": -1.5209734439849854, + "objective/rlhf_reward": 15.338401794433594, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009057885035872459, + "policy/clipfrac_avg": 0.009060706943273544, + "policy/entropy_avg": 0.161224365234375, + "step": 946, + "val/clipfrac_avg": 0.038856737315654755, + "val/num_eos_tokens": 409, + "val/ratio": 0.9891325235366821, + "val/ratio_var": 9.022479935083538e-05 + }, + { + "episode": 15152, + "epoch": 0.7652525252525253, + "eps": 0, + "loss/policy_avg": -0.051016077399253845, + "loss/value_avg": 0.5337303876876831, + "lr": 1.4456268380638262e-06, + "objective/entropy": -104.14939880371094, + "objective/kl": 36.21892547607422, + "objective/non_score_reward": -1.8109462261199951, + "objective/rlhf_reward": 15.282804489135742, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009193423902615905, + "policy/clipfrac_avg": 0.010075472295284271, + "policy/entropy_avg": 0.16665363311767578, + "step": 947, + "val/clipfrac_avg": 0.006654549390077591, + "val/num_eos_tokens": 424, + "val/ratio": 0.9888213276863098, + "val/ratio_var": 8.222387987188995e-05 + }, + { + "episode": 15168, + "epoch": 0.7660606060606061, + "eps": 0, + "loss/policy_avg": 0.5708428025245667, + "loss/value_avg": 0.9156904220581055, + "lr": 1.4362452251344283e-06, + "objective/entropy": -107.8341064453125, + "objective/kl": 37.6370849609375, + "objective/non_score_reward": -1.8818542957305908, + "objective/rlhf_reward": 11.461895942687988, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008800949435681105, + "policy/clipfrac_avg": 0.018943525850772858, + "policy/entropy_avg": 0.18418407440185547, + "step": 948, + "val/clipfrac_avg": 0.0006793478387407959, + "val/num_eos_tokens": 297, + "val/ratio": 0.9880671501159668, + "val/ratio_var": 7.767622446408495e-05 + }, + { + "episode": 15184, + "epoch": 0.7668686868686869, + "eps": 0, + "loss/policy_avg": -0.028068287298083305, + "loss/value_avg": 0.6219231486320496, + "lr": 1.4268890449160034e-06, + "objective/entropy": -11.566868782043457, + "objective/kl": 34.966495513916016, + "objective/non_score_reward": -1.7483247518539429, + "objective/rlhf_reward": 14.603238105773926, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0011735728476196527, + "policy/clipfrac_avg": 0.004941962193697691, + "policy/entropy_avg": 0.20639419555664062, + "step": 949, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 32, + "val/ratio": 0.9847333431243896, + "val/ratio_var": 3.736758299055509e-05 + }, + { + "episode": 15200, + "epoch": 0.7676767676767676, + "eps": 0, + "loss/policy_avg": -0.15076535940170288, + "loss/value_avg": 0.2775721549987793, + "lr": 1.4175583641788698e-06, + "objective/entropy": -80.4765625, + "objective/kl": 31.678754806518555, + "objective/non_score_reward": -1.583937644958496, + "objective/rlhf_reward": 13.853562355041504, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0006830325582996011, + "policy/clipfrac_avg": 0.00417131744325161, + "policy/entropy_avg": 0.1293621063232422, + "step": 950, + "val/clipfrac_avg": 0.0393822118639946, + "val/num_eos_tokens": 257, + "val/ratio": 0.9911376237869263, + "val/ratio_var": 2.9909415388829075e-05 + }, + { + "episode": 15216, + "epoch": 0.7684848484848484, + "eps": 0, + "loss/policy_avg": -0.04640840366482735, + "loss/value_avg": 0.3922756314277649, + "lr": 1.4082532495113627e-06, + "objective/entropy": -84.18096160888672, + "objective/kl": 38.69972229003906, + "objective/non_score_reward": -1.9349862337112427, + "objective/rlhf_reward": 15.416576385498047, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.000798642635345459, + "policy/clipfrac_avg": 0.0038171219639480114, + "policy/entropy_avg": 0.1776995062828064, + "step": 951, + "val/clipfrac_avg": 0.0032731681130826473, + "val/num_eos_tokens": 326, + "val/ratio": 0.9889781475067139, + "val/ratio_var": 5.715083170798607e-05 + }, + { + "episode": 15232, + "epoch": 0.7692929292929293, + "eps": 0, + "loss/policy_avg": -0.12606917321681976, + "loss/value_avg": 0.3175104856491089, + "lr": 1.3989737673193682e-06, + "objective/entropy": -60.43904113769531, + "objective/kl": 39.91594314575195, + "objective/non_score_reward": -1.9957971572875977, + "objective/rlhf_reward": 14.262016296386719, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009779632091522217, + "policy/clipfrac_avg": 0.004585625603795052, + "policy/entropy_avg": 0.2008209228515625, + "step": 952, + "val/clipfrac_avg": 0.030662208795547485, + "val/num_eos_tokens": 422, + "val/ratio": 0.9862244129180908, + "val/ratio_var": 8.592133963247761e-05 + }, + { + "episode": 15248, + "epoch": 0.7701010101010101, + "eps": 0, + "loss/policy_avg": 0.08768834173679352, + "loss/value_avg": 0.327705055475235, + "lr": 1.389719983825844e-06, + "objective/entropy": -67.14826965332031, + "objective/kl": 46.73610305786133, + "objective/non_score_reward": -2.3368051052093506, + "objective/rlhf_reward": 14.29600715637207, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011253968114033341, + "policy/clipfrac_avg": 0.006955197546631098, + "policy/entropy_avg": 0.20795369148254395, + "step": 953, + "val/clipfrac_avg": 8.926542068365961e-05, + "val/num_eos_tokens": 421, + "val/ratio": 0.9854748249053955, + "val/ratio_var": 6.634320016019046e-05 + }, + { + "episode": 15264, + "epoch": 0.7709090909090909, + "eps": 0, + "loss/policy_avg": 0.17172515392303467, + "loss/value_avg": 0.5182530283927917, + "lr": 1.3804919650703551e-06, + "objective/entropy": -52.92948913574219, + "objective/kl": 39.47554016113281, + "objective/non_score_reward": -1.9737770557403564, + "objective/rlhf_reward": 15.323097229003906, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009001094149425626, + "policy/clipfrac_avg": 0.005291195120662451, + "policy/entropy_avg": 0.18181610107421875, + "step": 954, + "val/clipfrac_avg": 0.006615057587623596, + "val/num_eos_tokens": 490, + "val/ratio": 0.9876183271408081, + "val/ratio_var": 5.796940240543336e-05 + }, + { + "episode": 15280, + "epoch": 0.7717171717171717, + "eps": 0, + "loss/policy_avg": 0.3471048176288605, + "loss/value_avg": 0.7186154723167419, + "lr": 1.3712897769085903e-06, + "objective/entropy": -93.25178527832031, + "objective/kl": 41.423980712890625, + "objective/non_score_reward": -2.0711989402770996, + "objective/rlhf_reward": 13.155363082885742, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009469924843870103, + "policy/clipfrac_avg": 0.004164956044405699, + "policy/entropy_avg": 0.18822860717773438, + "step": 955, + "val/clipfrac_avg": 0.04515676945447922, + "val/num_eos_tokens": 611, + "val/ratio": 0.9877511262893677, + "val/ratio_var": 7.300172001123428e-05 + }, + { + "episode": 15296, + "epoch": 0.7725252525252525, + "eps": 0, + "loss/policy_avg": 0.24007092416286469, + "loss/value_avg": 0.4516885280609131, + "lr": 1.3621134850119012e-06, + "objective/entropy": -112.50715637207031, + "objective/kl": 31.20224380493164, + "objective/non_score_reward": -1.5601121187210083, + "objective/rlhf_reward": 13.799263000488281, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007952903397381306, + "policy/clipfrac_avg": 0.008523119613528252, + "policy/entropy_avg": 0.1457352638244629, + "step": 956, + "val/clipfrac_avg": 0.06015515699982643, + "val/num_eos_tokens": 383, + "val/ratio": 0.9895913600921631, + "val/ratio_var": 7.848602399462834e-05 + }, + { + "episode": 15312, + "epoch": 0.7733333333333333, + "eps": 0, + "loss/policy_avg": -0.03721753507852554, + "loss/value_avg": 0.241817444562912, + "lr": 1.3529631548668298e-06, + "objective/entropy": -82.6330337524414, + "objective/kl": 44.714515686035156, + "objective/non_score_reward": -2.2357258796691895, + "objective/rlhf_reward": 13.818962097167969, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0006799673428758979, + "policy/clipfrac_avg": 0.003063515294343233, + "policy/entropy_avg": 0.1481151580810547, + "step": 957, + "val/clipfrac_avg": 9.245562250725925e-05, + "val/num_eos_tokens": 341, + "val/ratio": 0.9905509948730469, + "val/ratio_var": 4.010691918665543e-05 + }, + { + "episode": 15328, + "epoch": 0.7741414141414141, + "eps": 0, + "loss/policy_avg": -0.1448572874069214, + "loss/value_avg": 0.6361722946166992, + "lr": 1.3438388517746431e-06, + "objective/entropy": -75.73651885986328, + "objective/kl": 32.26376724243164, + "objective/non_score_reward": -1.6131882667541504, + "objective/rlhf_reward": 13.550874710083008, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010009508114308119, + "policy/clipfrac_avg": 0.004912985023111105, + "policy/entropy_avg": 0.190643310546875, + "step": 958, + "val/clipfrac_avg": 0.008336715400218964, + "val/num_eos_tokens": 329, + "val/ratio": 0.9864659905433655, + "val/ratio_var": 9.685769327916205e-05 + }, + { + "episode": 15344, + "epoch": 0.7749494949494949, + "eps": 0, + "loss/policy_avg": 0.11857907474040985, + "loss/value_avg": 0.8868049383163452, + "lr": 1.3347406408508695e-06, + "objective/entropy": -90.25643920898438, + "objective/kl": 39.02738952636719, + "objective/non_score_reward": -1.9513695240020752, + "objective/rlhf_reward": 14.954880714416504, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009018229902721941, + "policy/clipfrac_avg": 0.004778256639838219, + "policy/entropy_avg": 0.1796565055847168, + "step": 959, + "val/clipfrac_avg": 0.02706507220864296, + "val/num_eos_tokens": 289, + "val/ratio": 0.9882634878158569, + "val/ratio_var": 9.323142148787156e-05 + }, + { + "episode": 15360, + "epoch": 0.7757575757575758, + "eps": 0, + "loss/policy_avg": -0.22503328323364258, + "loss/value_avg": 0.4810031056404114, + "lr": 1.3256685870248227e-06, + "objective/entropy": -101.29171752929688, + "objective/kl": 37.34764099121094, + "objective/non_score_reward": -1.8673819303512573, + "objective/rlhf_reward": 14.32011890411377, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008543966105207801, + "policy/clipfrac_avg": 0.0047050644643604755, + "policy/entropy_avg": 0.17150402069091797, + "step": 960, + "val/clipfrac_avg": 0.0005405173869803548, + "val/num_eos_tokens": 130, + "val/ratio": 0.9882644414901733, + "val/ratio_var": 6.539086461998522e-05 + }, + { + "episode": 15376, + "epoch": 0.7765656565656566, + "eps": 0, + "loss/policy_avg": -0.0476013720035553, + "loss/value_avg": 0.682388961315155, + "lr": 1.3166227550391542e-06, + "objective/entropy": -97.00421142578125, + "objective/kl": 42.212120056152344, + "objective/non_score_reward": -2.1106061935424805, + "objective/rlhf_reward": 13.99876880645752, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000946523156017065, + "policy/clipfrac_avg": 0.0059087686240673065, + "policy/entropy_avg": 0.19147014617919922, + "step": 961, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 224, + "val/ratio": 0.9874411225318909, + "val/ratio_var": 5.3472565923584625e-05 + }, + { + "episode": 15392, + "epoch": 0.7773737373737374, + "eps": 0, + "loss/policy_avg": -0.26426494121551514, + "loss/value_avg": 0.47233304381370544, + "lr": 1.307603209449378e-06, + "objective/entropy": -60.28771209716797, + "objective/kl": 39.21135330200195, + "objective/non_score_reward": -1.9605677127838135, + "objective/rlhf_reward": 15.351932525634766, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009600625489838421, + "policy/clipfrac_avg": 0.0035959878005087376, + "policy/entropy_avg": 0.18239498138427734, + "step": 962, + "val/clipfrac_avg": 0.012040323577821255, + "val/num_eos_tokens": 296, + "val/ratio": 0.9875339269638062, + "val/ratio_var": 6.485576159320772e-05 + }, + { + "episode": 15408, + "epoch": 0.7781818181818182, + "eps": 0, + "loss/policy_avg": 0.04519583284854889, + "loss/value_avg": 0.7595553398132324, + "lr": 1.298610014623423e-06, + "objective/entropy": -146.63661193847656, + "objective/kl": 38.37220001220703, + "objective/non_score_reward": -1.9186100959777832, + "objective/rlhf_reward": 13.354827880859375, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007201317930594087, + "policy/clipfrac_avg": 0.0022899312898516655, + "policy/entropy_avg": 0.1562885046005249, + "step": 963, + "val/clipfrac_avg": 8.292270649690181e-05, + "val/num_eos_tokens": 521, + "val/ratio": 0.9906989336013794, + "val/ratio_var": 6.185618985909969e-05 + }, + { + "episode": 15424, + "epoch": 0.778989898989899, + "eps": 0, + "loss/policy_avg": -0.14623457193374634, + "loss/value_avg": 0.35101252794265747, + "lr": 1.2896432347411553e-06, + "objective/entropy": -66.08819580078125, + "objective/kl": 36.48185348510742, + "objective/non_score_reward": -1.8240927457809448, + "objective/rlhf_reward": 14.261844635009766, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009832159848883748, + "policy/clipfrac_avg": 0.006735666189342737, + "policy/entropy_avg": 0.19070911407470703, + "step": 964, + "val/clipfrac_avg": 0.0030241934582591057, + "val/num_eos_tokens": 421, + "val/ratio": 0.9872285723686218, + "val/ratio_var": 7.909470878075808e-05 + }, + { + "episode": 15440, + "epoch": 0.7797979797979798, + "eps": 0, + "loss/policy_avg": -0.1669466346502304, + "loss/value_avg": 0.6373587846755981, + "lr": 1.2807029337939403e-06, + "objective/entropy": -48.76942825317383, + "objective/kl": 36.51701354980469, + "objective/non_score_reward": -1.8258507251739502, + "objective/rlhf_reward": 13.861648559570312, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.000836203049402684, + "policy/clipfrac_avg": 0.006686762906610966, + "policy/entropy_avg": 0.17627906799316406, + "step": 965, + "val/clipfrac_avg": 0.009943181648850441, + "val/num_eos_tokens": 11, + "val/ratio": 0.9892607927322388, + "val/ratio_var": 5.405569390859455e-05 + }, + { + "episode": 15456, + "epoch": 0.7806060606060606, + "eps": 0, + "loss/policy_avg": -0.005087921395897865, + "loss/value_avg": 0.2712212800979614, + "lr": 1.2717891755841722e-06, + "objective/entropy": -59.34945297241211, + "objective/kl": 42.5623893737793, + "objective/non_score_reward": -2.128119468688965, + "objective/rlhf_reward": 13.918755531311035, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008710323600098491, + "policy/clipfrac_avg": 0.010207775980234146, + "policy/entropy_avg": 0.1739978790283203, + "step": 966, + "val/clipfrac_avg": 0.010802468284964561, + "val/num_eos_tokens": 246, + "val/ratio": 0.9884668588638306, + "val/ratio_var": 4.567186624626629e-05 + }, + { + "episode": 15472, + "epoch": 0.7814141414141414, + "eps": 0, + "loss/policy_avg": -0.5036620497703552, + "loss/value_avg": 0.39513152837753296, + "lr": 1.2629020237248241e-06, + "objective/entropy": -136.59652709960938, + "objective/kl": 29.273542404174805, + "objective/non_score_reward": -1.463677167892456, + "objective/rlhf_reward": 14.083197593688965, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0007826394285075366, + "policy/clipfrac_avg": 0.006299240980297327, + "policy/entropy_avg": 0.15417519211769104, + "step": 967, + "val/clipfrac_avg": 0.03271849453449249, + "val/num_eos_tokens": 249, + "val/ratio": 0.9902065992355347, + "val/ratio_var": 8.734448783798143e-05 + }, + { + "episode": 15488, + "epoch": 0.7822222222222223, + "eps": 0, + "loss/policy_avg": -0.04834941402077675, + "loss/value_avg": 0.49886178970336914, + "lr": 1.2540415416389917e-06, + "objective/entropy": -29.31902503967285, + "objective/kl": 38.655548095703125, + "objective/non_score_reward": -1.9327776432037354, + "objective/rlhf_reward": 13.864096641540527, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008980755810625851, + "policy/clipfrac_avg": 0.003656789194792509, + "policy/entropy_avg": 0.1847667694091797, + "step": 968, + "val/clipfrac_avg": 5.3879310144111514e-05, + "val/num_eos_tokens": 10, + "val/ratio": 0.9879658818244934, + "val/ratio_var": 2.3956074073794298e-05 + }, + { + "episode": 15504, + "epoch": 0.7830303030303031, + "eps": 0, + "loss/policy_avg": -0.2217503786087036, + "loss/value_avg": 0.4100498557090759, + "lr": 1.2452077925594435e-06, + "objective/entropy": -206.95504760742188, + "objective/kl": 41.82144546508789, + "objective/non_score_reward": -2.0910723209381104, + "objective/rlhf_reward": 13.076896667480469, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008509804611094296, + "policy/clipfrac_avg": 0.0047798133455216885, + "policy/entropy_avg": 0.15673446655273438, + "step": 969, + "val/clipfrac_avg": 0.00021062050655018538, + "val/num_eos_tokens": 481, + "val/ratio": 0.9887630939483643, + "val/ratio_var": 6.604092777706683e-05 + }, + { + "episode": 15520, + "epoch": 0.7838383838383839, + "eps": 0, + "loss/policy_avg": -0.02120068296790123, + "loss/value_avg": 0.6901993751525879, + "lr": 1.23640083952817e-06, + "objective/entropy": -40.900238037109375, + "objective/kl": 38.07307052612305, + "objective/non_score_reward": -1.9036535024642944, + "objective/rlhf_reward": 14.377596855163574, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010727059561759233, + "policy/clipfrac_avg": 0.004337977152317762, + "policy/entropy_avg": 0.1984100341796875, + "step": 970, + "val/clipfrac_avg": 0.02858383022248745, + "val/num_eos_tokens": 535, + "val/ratio": 0.9863028526306152, + "val/ratio_var": 8.229557715822011e-05 + }, + { + "episode": 15536, + "epoch": 0.7846464646464646, + "eps": 0, + "loss/policy_avg": -0.09917282313108444, + "loss/value_avg": 0.3991038203239441, + "lr": 1.2276207453959283e-06, + "objective/entropy": -77.6534652709961, + "objective/kl": 39.35569381713867, + "objective/non_score_reward": -1.9677847623825073, + "objective/rlhf_reward": 13.907215118408203, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008703683270141482, + "policy/clipfrac_avg": 0.0043828776106238365, + "policy/entropy_avg": 0.17210960388183594, + "step": 971, + "val/clipfrac_avg": 0.00012400794366840273, + "val/num_eos_tokens": 175, + "val/ratio": 0.988074541091919, + "val/ratio_var": 5.318454350344837e-05 + }, + { + "episode": 15552, + "epoch": 0.7854545454545454, + "eps": 0, + "loss/policy_avg": 0.0814996063709259, + "loss/value_avg": 0.6175538897514343, + "lr": 1.2188675728217986e-06, + "objective/entropy": -16.92827606201172, + "objective/kl": 41.14687728881836, + "objective/non_score_reward": -2.0573439598083496, + "objective/rlhf_reward": 15.442655563354492, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010900306515395641, + "policy/clipfrac_avg": 0.00866631604731083, + "policy/entropy_avg": 0.2063446044921875, + "step": 972, + "val/clipfrac_avg": 0.009025804698467255, + "val/num_eos_tokens": 174, + "val/ratio": 0.9854716062545776, + "val/ratio_var": 2.7480951757752337e-05 + }, + { + "episode": 15568, + "epoch": 0.7862626262626262, + "eps": 0, + "loss/policy_avg": 0.3593114912509918, + "loss/value_avg": 0.7372395396232605, + "lr": 1.2101413842727345e-06, + "objective/entropy": -63.132266998291016, + "objective/kl": 45.04676818847656, + "objective/non_score_reward": -2.252338409423828, + "objective/rlhf_reward": 14.450786590576172, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009616734459996223, + "policy/clipfrac_avg": 0.0046102688647806644, + "policy/entropy_avg": 0.1856832504272461, + "step": 973, + "val/clipfrac_avg": 0.016889449208974838, + "val/num_eos_tokens": 419, + "val/ratio": 0.9877485036849976, + "val/ratio_var": 6.443963502533734e-05 + }, + { + "episode": 15584, + "epoch": 0.787070707070707, + "eps": 0, + "loss/policy_avg": 0.14888805150985718, + "loss/value_avg": 0.7544107437133789, + "lr": 1.201442242023122e-06, + "objective/entropy": -98.38099670410156, + "objective/kl": 41.9028434753418, + "objective/non_score_reward": -2.095142126083374, + "objective/rlhf_reward": 12.186107635498047, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009328118758276105, + "policy/clipfrac_avg": 0.004906289279460907, + "policy/entropy_avg": 0.17488479614257812, + "step": 974, + "val/clipfrac_avg": 0.08914318680763245, + "val/num_eos_tokens": 193, + "val/ratio": 0.9888035655021667, + "val/ratio_var": 6.763465353287756e-05 + }, + { + "episode": 15600, + "epoch": 0.7878787878787878, + "eps": 0, + "loss/policy_avg": -0.05416979640722275, + "loss/value_avg": 0.5626612901687622, + "lr": 1.1927702081543279e-06, + "objective/entropy": -65.54731750488281, + "objective/kl": 33.80537796020508, + "objective/non_score_reward": -1.6902689933776855, + "objective/rlhf_reward": 13.434730529785156, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010051055578514934, + "policy/clipfrac_avg": 0.005396673921495676, + "policy/entropy_avg": 0.19644975662231445, + "step": 975, + "val/clipfrac_avg": 0.02293882891535759, + "val/num_eos_tokens": 539, + "val/ratio": 0.9861206412315369, + "val/ratio_var": 7.48153543099761e-05 + }, + { + "episode": 15616, + "epoch": 0.7886868686868687, + "eps": 0, + "loss/policy_avg": 0.05607307702302933, + "loss/value_avg": 0.48446062207221985, + "lr": 1.1841253445542605e-06, + "objective/entropy": -29.001220703125, + "objective/kl": 36.37082290649414, + "objective/non_score_reward": -1.8185412883758545, + "objective/rlhf_reward": 12.142396926879883, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0011397147318348289, + "policy/clipfrac_avg": 0.0031259057577699423, + "policy/entropy_avg": 0.22170257568359375, + "step": 976, + "val/clipfrac_avg": 0.005424283444881439, + "val/num_eos_tokens": 387, + "val/ratio": 0.9855947494506836, + "val/ratio_var": 6.33225790807046e-05 + }, + { + "episode": 15632, + "epoch": 0.7894949494949495, + "eps": 0, + "loss/policy_avg": -0.15400168299674988, + "loss/value_avg": 0.8165977001190186, + "lr": 1.1755077129169284e-06, + "objective/entropy": -60.56960678100586, + "objective/kl": 37.47486114501953, + "objective/non_score_reward": -1.8737430572509766, + "objective/rlhf_reward": 11.046178817749023, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0009815995581448078, + "policy/clipfrac_avg": 0.005213981028646231, + "policy/entropy_avg": 0.21601486206054688, + "step": 977, + "val/clipfrac_avg": 0.022167012095451355, + "val/num_eos_tokens": 93, + "val/ratio": 0.9861747026443481, + "val/ratio_var": 7.86496966611594e-05 + }, + { + "episode": 15648, + "epoch": 0.7903030303030303, + "eps": 0, + "loss/policy_avg": -0.1725654900074005, + "loss/value_avg": 0.487238347530365, + "lr": 1.166917374742e-06, + "objective/entropy": -56.807586669921875, + "objective/kl": 37.225948333740234, + "objective/non_score_reward": -1.861297607421875, + "objective/rlhf_reward": 14.388702392578125, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.00102808338124305, + "policy/clipfrac_avg": 0.0037505230866372585, + "policy/entropy_avg": 0.20328998565673828, + "step": 978, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 128, + "val/ratio": 0.9871898293495178, + "val/ratio_var": 9.372083150083199e-05 + }, + { + "episode": 15664, + "epoch": 0.7911111111111111, + "eps": 0, + "loss/policy_avg": -0.2764543294906616, + "loss/value_avg": 0.33185577392578125, + "lr": 1.158354391334362e-06, + "objective/entropy": -78.30638122558594, + "objective/kl": 34.59331512451172, + "objective/non_score_reward": -1.729665756225586, + "objective/rlhf_reward": 15.489084243774414, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008904373971745372, + "policy/clipfrac_avg": 0.00305290799587965, + "policy/entropy_avg": 0.17419052124023438, + "step": 979, + "val/clipfrac_avg": 0.09470640122890472, + "val/num_eos_tokens": 367, + "val/ratio": 0.9881394505500793, + "val/ratio_var": 6.588637916138396e-05 + }, + { + "episode": 15680, + "epoch": 0.7919191919191919, + "eps": 0, + "loss/policy_avg": 0.049546245485544205, + "loss/value_avg": 0.5992924571037292, + "lr": 1.1498188238036862e-06, + "objective/entropy": -146.6639862060547, + "objective/kl": 40.94211959838867, + "objective/non_score_reward": -2.0471062660217285, + "objective/rlhf_reward": 12.40992546081543, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008619745494797826, + "policy/clipfrac_avg": 0.007621705532073975, + "policy/entropy_avg": 0.15411758422851562, + "step": 980, + "val/clipfrac_avg": 0.09697519987821579, + "val/num_eos_tokens": 405, + "val/ratio": 0.9889684915542603, + "val/ratio_var": 5.157575287739746e-05 + }, + { + "episode": 15696, + "epoch": 0.7927272727272727, + "eps": 0, + "loss/policy_avg": 0.7101428508758545, + "loss/value_avg": 1.0345678329467773, + "lr": 1.141310733063991e-06, + "objective/entropy": -111.55574798583984, + "objective/kl": 35.79730224609375, + "objective/non_score_reward": -1.7898651361465454, + "objective/rlhf_reward": 13.764822006225586, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009224195964634418, + "policy/clipfrac_avg": 0.00732937827706337, + "policy/entropy_avg": 0.1737055778503418, + "step": 981, + "val/clipfrac_avg": 0.0015625000232830644, + "val/num_eos_tokens": 230, + "val/ratio": 0.9885663390159607, + "val/ratio_var": 7.395897409878671e-05 + }, + { + "episode": 15712, + "epoch": 0.7935353535353535, + "eps": 0, + "loss/policy_avg": -0.1941525638103485, + "loss/value_avg": 0.4167664647102356, + "lr": 1.132830179833207e-06, + "objective/entropy": -136.7578125, + "objective/kl": 41.060279846191406, + "objective/non_score_reward": -2.053014039993286, + "objective/rlhf_reward": 14.564172744750977, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008562670554965734, + "policy/clipfrac_avg": 0.005737176164984703, + "policy/entropy_avg": 0.1706523895263672, + "step": 982, + "val/clipfrac_avg": 0.006640523672103882, + "val/num_eos_tokens": 935, + "val/ratio": 0.9893268346786499, + "val/ratio_var": 6.982601917115971e-05 + }, + { + "episode": 15728, + "epoch": 0.7943434343434344, + "eps": 0, + "loss/policy_avg": -0.1428111046552658, + "loss/value_avg": 0.2174670249223709, + "lr": 1.1243772246327416e-06, + "objective/entropy": -109.03129577636719, + "objective/kl": 38.72102355957031, + "objective/non_score_reward": -1.9360511302947998, + "objective/rlhf_reward": 14.470198631286621, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000771217979490757, + "policy/clipfrac_avg": 0.002998888725414872, + "policy/entropy_avg": 0.16396713256835938, + "step": 983, + "val/clipfrac_avg": 0.020510124042630196, + "val/num_eos_tokens": 25, + "val/ratio": 0.989021897315979, + "val/ratio_var": 4.0093269490171224e-05 + }, + { + "episode": 15744, + "epoch": 0.7951515151515152, + "eps": 0, + "loss/policy_avg": 0.010445185005664825, + "loss/value_avg": 0.37888824939727783, + "lr": 1.1159519277870507e-06, + "objective/entropy": -6.635685920715332, + "objective/kl": 38.89892578125, + "objective/non_score_reward": -1.9449464082717896, + "objective/rlhf_reward": 14.7113037109375, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009513833792880177, + "policy/clipfrac_avg": 0.003157425671815872, + "policy/entropy_avg": 0.1973400115966797, + "step": 984, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 21, + "val/ratio": 0.9875931739807129, + "val/ratio_var": 5.873597910976969e-05 + }, + { + "episode": 15760, + "epoch": 0.795959595959596, + "eps": 0, + "loss/policy_avg": -0.28500208258628845, + "loss/value_avg": 0.4213254451751709, + "lr": 1.1075543494232049e-06, + "objective/entropy": -149.3587646484375, + "objective/kl": 31.575908660888672, + "objective/non_score_reward": -1.5787954330444336, + "objective/rlhf_reward": 15.75714111328125, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007162499241530895, + "policy/clipfrac_avg": 0.0078894617035985, + "policy/entropy_avg": 0.14484691619873047, + "step": 985, + "val/clipfrac_avg": 0.023666253313422203, + "val/num_eos_tokens": 155, + "val/ratio": 0.9908950328826904, + "val/ratio_var": 5.517086901818402e-05 + }, + { + "episode": 15776, + "epoch": 0.7967676767676768, + "eps": 0, + "loss/policy_avg": -0.016947008669376373, + "loss/value_avg": 0.3959222733974457, + "lr": 1.0991845494704673e-06, + "objective/entropy": -90.30593872070312, + "objective/kl": 44.143898010253906, + "objective/non_score_reward": -2.2071948051452637, + "objective/rlhf_reward": 12.292805671691895, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000890448980499059, + "policy/clipfrac_avg": 0.005722125992178917, + "policy/entropy_avg": 0.1587371826171875, + "step": 986, + "val/clipfrac_avg": 0.020597470924258232, + "val/num_eos_tokens": 327, + "val/ratio": 0.9887195825576782, + "val/ratio_var": 5.094557855045423e-05 + }, + { + "episode": 15792, + "epoch": 0.7975757575757576, + "eps": 0, + "loss/policy_avg": 0.022888287901878357, + "loss/value_avg": 0.36602234840393066, + "lr": 1.0908425876598512e-06, + "objective/entropy": -140.4935760498047, + "objective/kl": 42.647743225097656, + "objective/non_score_reward": -2.132387161254883, + "objective/rlhf_reward": 13.516050338745117, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008821362862363458, + "policy/clipfrac_avg": 0.005631787236779928, + "policy/entropy_avg": 0.1700587272644043, + "step": 987, + "val/clipfrac_avg": 0.012090040370821953, + "val/num_eos_tokens": 546, + "val/ratio": 0.9886399507522583, + "val/ratio_var": 5.682192204403691e-05 + }, + { + "episode": 15808, + "epoch": 0.7983838383838384, + "eps": 0, + "loss/policy_avg": 0.19011469185352325, + "loss/value_avg": 0.5114719271659851, + "lr": 1.082528523523713e-06, + "objective/entropy": -53.488460540771484, + "objective/kl": 36.984073638916016, + "objective/non_score_reward": -1.8492038249969482, + "objective/rlhf_reward": 14.049233436584473, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009789207251742482, + "policy/clipfrac_avg": 0.005884348414838314, + "policy/entropy_avg": 0.20690059661865234, + "step": 988, + "val/clipfrac_avg": 0.013353430666029453, + "val/num_eos_tokens": 353, + "val/ratio": 0.9872037768363953, + "val/ratio_var": 6.569022662006319e-05 + }, + { + "episode": 15824, + "epoch": 0.7991919191919192, + "eps": 0, + "loss/policy_avg": 0.1753331869840622, + "loss/value_avg": 0.3773725628852844, + "lr": 1.0742424163953096e-06, + "objective/entropy": -56.68682861328125, + "objective/kl": 37.34782409667969, + "objective/non_score_reward": -1.8673913478851318, + "objective/rlhf_reward": 13.050577163696289, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.000838876876514405, + "policy/clipfrac_avg": 0.0064134905114769936, + "policy/entropy_avg": 0.16259098052978516, + "step": 989, + "val/clipfrac_avg": 0.055415935814380646, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885395169258118, + "val/ratio_var": 8.253273699665442e-05 + }, + { + "episode": 15840, + "epoch": 0.8, + "eps": 0, + "loss/policy_avg": -0.013544011861085892, + "loss/value_avg": 0.5490726828575134, + "lr": 1.0659843254083919e-06, + "objective/entropy": -84.58714294433594, + "objective/kl": 34.28784942626953, + "objective/non_score_reward": -1.7143926620483398, + "objective/rlhf_reward": 13.20748233795166, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009669119026511908, + "policy/clipfrac_avg": 0.00355747202411294, + "policy/entropy_avg": 0.18613052368164062, + "step": 990, + "val/clipfrac_avg": 0.015838205814361572, + "val/num_eos_tokens": 115, + "val/ratio": 0.9877301454544067, + "val/ratio_var": 5.967232209513895e-05 + }, + { + "episode": 15856, + "epoch": 0.8008080808080809, + "eps": 0, + "loss/policy_avg": 0.17707303166389465, + "loss/value_avg": 0.24937717616558075, + "lr": 1.0577543094967613e-06, + "objective/entropy": -134.7545928955078, + "objective/kl": 35.57737731933594, + "objective/non_score_reward": -1.7788686752319336, + "objective/rlhf_reward": 15.010193824768066, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008520459523424506, + "policy/clipfrac_avg": 0.007036273833364248, + "policy/entropy_avg": 0.14983177185058594, + "step": 991, + "val/clipfrac_avg": 0.0005040322430431843, + "val/num_eos_tokens": 450, + "val/ratio": 0.9898406863212585, + "val/ratio_var": 4.8867601435631514e-05 + }, + { + "episode": 15872, + "epoch": 0.8016161616161617, + "eps": 0, + "loss/policy_avg": -0.1231779083609581, + "loss/value_avg": 0.27385565638542175, + "lr": 1.0495524273938735e-06, + "objective/entropy": -99.90908813476562, + "objective/kl": 37.30508804321289, + "objective/non_score_reward": -1.865254521369934, + "objective/rlhf_reward": 12.759745597839355, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008709601243026555, + "policy/clipfrac_avg": 0.004131123423576355, + "policy/entropy_avg": 0.17008209228515625, + "step": 992, + "val/clipfrac_avg": 0.01071952749043703, + "val/num_eos_tokens": 376, + "val/ratio": 0.9894505739212036, + "val/ratio_var": 6.905445479787886e-05 + }, + { + "episode": 15888, + "epoch": 0.8024242424242424, + "eps": 0, + "loss/policy_avg": -0.11878392100334167, + "loss/value_avg": 0.6012347340583801, + "lr": 1.041378737632402e-06, + "objective/entropy": -89.06427764892578, + "objective/kl": 39.19951629638672, + "objective/non_score_reward": -1.9599759578704834, + "objective/rlhf_reward": 14.532211303710938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009216031758114696, + "policy/clipfrac_avg": 0.00672511849552393, + "policy/entropy_avg": 0.1765270233154297, + "step": 993, + "val/clipfrac_avg": 0.0031250000465661287, + "val/num_eos_tokens": 74, + "val/ratio": 0.9882692098617554, + "val/ratio_var": 3.506496432237327e-05 + }, + { + "episode": 15904, + "epoch": 0.8032323232323232, + "eps": 0, + "loss/policy_avg": 0.11932457238435745, + "loss/value_avg": 0.28720468282699585, + "lr": 1.0332332985438248e-06, + "objective/entropy": -48.50525665283203, + "objective/kl": 37.94336700439453, + "objective/non_score_reward": -1.8971682786941528, + "objective/rlhf_reward": 15.641894340515137, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0010036650346592069, + "policy/clipfrac_avg": 0.0061247204430401325, + "policy/entropy_avg": 0.18225669860839844, + "step": 994, + "val/clipfrac_avg": 0.01019021775573492, + "val/num_eos_tokens": 8, + "val/ratio": 0.9874961972236633, + "val/ratio_var": 5.3736544941784814e-05 + }, + { + "episode": 15920, + "epoch": 0.804040404040404, + "eps": 0, + "loss/policy_avg": 0.14462614059448242, + "loss/value_avg": 0.3247252106666565, + "lr": 1.0251161682580125e-06, + "objective/entropy": -108.57736206054688, + "objective/kl": 41.1889762878418, + "objective/non_score_reward": -2.059448719024658, + "objective/rlhf_reward": 14.182738304138184, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001018329057842493, + "policy/clipfrac_avg": 0.0034168935380876064, + "policy/entropy_avg": 0.20872879028320312, + "step": 995, + "val/clipfrac_avg": 0.00548538425937295, + "val/num_eos_tokens": 8, + "val/ratio": 0.9865425825119019, + "val/ratio_var": 8.348035044036806e-05 + }, + { + "episode": 15936, + "epoch": 0.8048484848484848, + "eps": 0, + "loss/policy_avg": 0.2817779779434204, + "loss/value_avg": 0.49794071912765503, + "lr": 1.0170274047028068e-06, + "objective/entropy": -43.08762741088867, + "objective/kl": 45.49590301513672, + "objective/non_score_reward": -2.2747950553894043, + "objective/rlhf_reward": 13.912704467773438, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010373364202678204, + "policy/clipfrac_avg": 0.006608482450246811, + "policy/entropy_avg": 0.2166609764099121, + "step": 996, + "val/clipfrac_avg": 0.03890983760356903, + "val/num_eos_tokens": 11, + "val/ratio": 0.986150860786438, + "val/ratio_var": 8.553406951250508e-05 + }, + { + "episode": 15952, + "epoch": 0.8056565656565656, + "eps": 0, + "loss/policy_avg": -0.22047999501228333, + "loss/value_avg": 0.5745018720626831, + "lr": 1.0089670656036137e-06, + "objective/entropy": -114.1541976928711, + "objective/kl": 35.899574279785156, + "objective/non_score_reward": -1.7949787378311157, + "objective/rlhf_reward": 14.869083404541016, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007367507787421346, + "policy/clipfrac_avg": 0.0037586798425763845, + "policy/entropy_avg": 0.15613555908203125, + "step": 997, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 560, + "val/ratio": 0.9902479648590088, + "val/ratio_var": 5.6676559324841946e-05 + }, + { + "episode": 15968, + "epoch": 0.8064646464646464, + "eps": 0, + "loss/policy_avg": -0.04391951486468315, + "loss/value_avg": 0.6516869068145752, + "lr": 1.0009352084829871e-06, + "objective/entropy": -112.42098999023438, + "objective/kl": 39.436988830566406, + "objective/non_score_reward": -1.9718495607376099, + "objective/rlhf_reward": 14.48127555847168, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009028672939166427, + "policy/clipfrac_avg": 0.0059207589365541935, + "policy/entropy_avg": 0.18039512634277344, + "step": 998, + "val/clipfrac_avg": 0.0127427838742733, + "val/num_eos_tokens": 515, + "val/ratio": 0.9878066778182983, + "val/ratio_var": 8.427277498412877e-05 + }, + { + "episode": 15984, + "epoch": 0.8072727272727273, + "eps": 0, + "loss/policy_avg": -0.5726251602172852, + "loss/value_avg": 0.6322129964828491, + "lr": 9.929318906602176e-07, + "objective/entropy": -115.74237060546875, + "objective/kl": 35.111915588378906, + "objective/non_score_reward": -1.7555959224700928, + "objective/rlhf_reward": 14.103778839111328, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006586643867194653, + "policy/clipfrac_avg": 0.006481423042714596, + "policy/entropy_avg": 0.1394972801208496, + "step": 999, + "val/clipfrac_avg": 0.019941631704568863, + "val/num_eos_tokens": 81, + "val/ratio": 0.9906902313232422, + "val/ratio_var": 5.2608244004659355e-05 + }, + { + "episode": 16000, + "epoch": 0.8080808080808081, + "eps": 0, + "loss/policy_avg": -0.1571139395236969, + "loss/value_avg": 0.8596901297569275, + "lr": 9.849571692509257e-07, + "objective/entropy": -35.66822052001953, + "objective/kl": 36.119667053222656, + "objective/non_score_reward": -1.805983304977417, + "objective/rlhf_reward": 12.842453956604004, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008497819071635604, + "policy/clipfrac_avg": 0.0041199941188097, + "policy/entropy_avg": 0.16556644439697266, + "step": 1000, + "val/clipfrac_avg": 0.022714650258421898, + "val/num_eos_tokens": 82, + "val/ratio": 0.9879409074783325, + "val/ratio_var": 6.342474807752296e-05 + }, + { + "episode": 16016, + "epoch": 0.8088888888888889, + "eps": 0, + "loss/policy_avg": -0.10610047727823257, + "loss/value_avg": 0.5593327283859253, + "lr": 9.770111011666582e-07, + "objective/entropy": -112.11347961425781, + "objective/kl": 33.91107177734375, + "objective/non_score_reward": -1.6955535411834717, + "objective/rlhf_reward": 13.304447174072266, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008447067812085152, + "policy/clipfrac_avg": 0.004359187558293343, + "policy/entropy_avg": 0.18085873126983643, + "step": 1001, + "val/clipfrac_avg": 0.0036525975447148085, + "val/num_eos_tokens": 37, + "val/ratio": 0.9888993501663208, + "val/ratio_var": 0.00012090770906070247 + }, + { + "episode": 16032, + "epoch": 0.8096969696969697, + "eps": 0, + "loss/policy_avg": -0.4011276960372925, + "loss/value_avg": 0.8269133567810059, + "lr": 9.690937431144725e-07, + "objective/entropy": -143.82159423828125, + "objective/kl": 32.61180877685547, + "objective/non_score_reward": -1.6305904388427734, + "objective/rlhf_reward": 13.322534561157227, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0007577177020721138, + "policy/clipfrac_avg": 0.005468646064400673, + "policy/entropy_avg": 0.14389371871948242, + "step": 1002, + "val/clipfrac_avg": 0.016304347664117813, + "val/num_eos_tokens": 153, + "val/ratio": 0.990355908870697, + "val/ratio_var": 4.9998925533145666e-05 + }, + { + "episode": 16048, + "epoch": 0.8105050505050505, + "eps": 0, + "loss/policy_avg": -0.18046468496322632, + "loss/value_avg": 0.4406319260597229, + "lr": 9.612051515965388e-07, + "objective/entropy": -41.019737243652344, + "objective/kl": 43.84602355957031, + "objective/non_score_reward": -2.1923012733459473, + "objective/rlhf_reward": 14.870198249816895, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009248010464943945, + "policy/clipfrac_avg": 0.004084739368408918, + "policy/entropy_avg": 0.1930408477783203, + "step": 1003, + "val/clipfrac_avg": 0.01253926008939743, + "val/num_eos_tokens": 425, + "val/ratio": 0.9877877235412598, + "val/ratio_var": 4.6758981625316665e-05 + }, + { + "episode": 16064, + "epoch": 0.8113131313131313, + "eps": 0, + "loss/policy_avg": -0.03701537847518921, + "loss/value_avg": 0.4312940835952759, + "lr": 9.533453829097355e-07, + "objective/entropy": -115.09342956542969, + "objective/kl": 43.825538635253906, + "objective/non_score_reward": -2.191277027130127, + "objective/rlhf_reward": 14.941534996032715, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008623325265944004, + "policy/clipfrac_avg": 0.004582014866173267, + "policy/entropy_avg": 0.17541944980621338, + "step": 1004, + "val/clipfrac_avg": 0.02018960565328598, + "val/num_eos_tokens": 259, + "val/ratio": 0.988018274307251, + "val/ratio_var": 5.916683949180879e-05 + }, + { + "episode": 16080, + "epoch": 0.8121212121212121, + "eps": 0, + "loss/policy_avg": 0.30280348658561707, + "loss/value_avg": 0.7898949384689331, + "lr": 9.455144931452459e-07, + "objective/entropy": -62.099971771240234, + "objective/kl": 30.437484741210938, + "objective/non_score_reward": -1.5218743085861206, + "objective/rlhf_reward": 13.712501525878906, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007971136947162449, + "policy/clipfrac_avg": 0.008183586411178112, + "policy/entropy_avg": 0.1460132598876953, + "step": 1005, + "val/clipfrac_avg": 0.003989361692219973, + "val/num_eos_tokens": 215, + "val/ratio": 0.9896875023841858, + "val/ratio_var": 4.330440060584806e-05 + }, + { + "episode": 16096, + "epoch": 0.812929292929293, + "eps": 0, + "loss/policy_avg": -0.23599471151828766, + "loss/value_avg": 1.1437101364135742, + "lr": 9.377125381881607e-07, + "objective/entropy": -46.674827575683594, + "objective/kl": 33.718963623046875, + "objective/non_score_reward": -1.6859480142593384, + "objective/rlhf_reward": 14.86874008178711, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.00085799815133214, + "policy/clipfrac_avg": 0.003210378112271428, + "policy/entropy_avg": 0.19154703617095947, + "step": 1006, + "val/clipfrac_avg": 0.010907451622188091, + "val/num_eos_tokens": 259, + "val/ratio": 0.9871425628662109, + "val/ratio_var": 6.926390778971836e-05 + }, + { + "episode": 16112, + "epoch": 0.8137373737373738, + "eps": 0, + "loss/policy_avg": -0.02023145742714405, + "loss/value_avg": 1.2994359731674194, + "lr": 9.299395737170758e-07, + "objective/entropy": -87.60648345947266, + "objective/kl": 35.53752899169922, + "objective/non_score_reward": -1.7768765687942505, + "objective/rlhf_reward": 15.644998550415039, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0007149496814236045, + "policy/clipfrac_avg": 0.002271715085953474, + "policy/entropy_avg": 0.1492328643798828, + "step": 1007, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 12, + "val/ratio": 0.9887728691101074, + "val/ratio_var": 5.388016870710999e-05 + }, + { + "episode": 16128, + "epoch": 0.8145454545454546, + "eps": 0, + "loss/policy_avg": 0.022628188133239746, + "loss/value_avg": 1.1084355115890503, + "lr": 9.221956552036992e-07, + "objective/entropy": -63.064239501953125, + "objective/kl": 36.12407302856445, + "objective/non_score_reward": -1.8062036037445068, + "objective/rlhf_reward": 15.146921157836914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008970820344984531, + "policy/clipfrac_avg": 0.004715266637504101, + "policy/entropy_avg": 0.1754002571105957, + "step": 1008, + "val/clipfrac_avg": 0.008854166604578495, + "val/num_eos_tokens": 540, + "val/ratio": 0.9880603551864624, + "val/ratio_var": 8.214469562517479e-05 + }, + { + "episode": 16144, + "epoch": 0.8153535353535354, + "eps": 0, + "loss/policy_avg": -0.06129603087902069, + "loss/value_avg": 0.558459997177124, + "lr": 9.144808379124509e-07, + "objective/entropy": -104.08320617675781, + "objective/kl": 38.30896759033203, + "objective/non_score_reward": -1.915448546409607, + "objective/rlhf_reward": 13.201739311218262, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008289464749395847, + "policy/clipfrac_avg": 0.0036508054472506046, + "policy/entropy_avg": 0.16502058506011963, + "step": 1009, + "val/clipfrac_avg": 0.013535099104046822, + "val/num_eos_tokens": 360, + "val/ratio": 0.9892135858535767, + "val/ratio_var": 6.506205681944266e-05 + }, + { + "episode": 16160, + "epoch": 0.8161616161616162, + "eps": 0, + "loss/policy_avg": -0.06057693436741829, + "loss/value_avg": 0.288950115442276, + "lr": 9.067951769000699e-07, + "objective/entropy": -71.51614379882812, + "objective/kl": 42.83964538574219, + "objective/non_score_reward": -2.141982078552246, + "objective/rlhf_reward": 14.701767921447754, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009444506140425801, + "policy/clipfrac_avg": 0.002944172825664282, + "policy/entropy_avg": 0.1825122833251953, + "step": 1010, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 613, + "val/ratio": 0.9874241352081299, + "val/ratio_var": 3.9652099076192826e-05 + }, + { + "episode": 16176, + "epoch": 0.816969696969697, + "eps": 0, + "loss/policy_avg": -0.0031431764364242554, + "loss/value_avg": 0.5874146223068237, + "lr": 8.991387270152202e-07, + "objective/entropy": -63.48155212402344, + "objective/kl": 43.46244812011719, + "objective/non_score_reward": -2.1731224060058594, + "objective/rlhf_reward": 14.65500259399414, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009073936380445957, + "policy/clipfrac_avg": 0.005083385389298201, + "policy/entropy_avg": 0.1724100112915039, + "step": 1011, + "val/clipfrac_avg": 0.0003350692568346858, + "val/num_eos_tokens": 564, + "val/ratio": 0.9878172874450684, + "val/ratio_var": 5.2520466852001846e-05 + }, + { + "episode": 16192, + "epoch": 0.8177777777777778, + "eps": 0, + "loss/policy_avg": 0.020213183015584946, + "loss/value_avg": 0.3557816445827484, + "lr": 8.915115428981064e-07, + "objective/entropy": -24.23276138305664, + "objective/kl": 30.50366973876953, + "objective/non_score_reward": -1.5251834392547607, + "objective/rlhf_reward": 14.45919132232666, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009571783011779189, + "policy/clipfrac_avg": 0.004434539936482906, + "policy/entropy_avg": 0.18828392028808594, + "step": 1012, + "val/clipfrac_avg": 0.015398487448692322, + "val/num_eos_tokens": 104, + "val/ratio": 0.9878268241882324, + "val/ratio_var": 6.845602911198512e-05 + }, + { + "episode": 16208, + "epoch": 0.8185858585858586, + "eps": 0, + "loss/policy_avg": -0.06331466138362885, + "loss/value_avg": 0.6016228795051575, + "lr": 8.839136789800729e-07, + "objective/entropy": -72.25382995605469, + "objective/kl": 35.868072509765625, + "objective/non_score_reward": -1.7934037446975708, + "objective/rlhf_reward": 14.183158874511719, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008664189372211695, + "policy/clipfrac_avg": 0.0032933237962424755, + "policy/entropy_avg": 0.17766380310058594, + "step": 1013, + "val/clipfrac_avg": 0.006926534231752157, + "val/num_eos_tokens": 294, + "val/ratio": 0.987805962562561, + "val/ratio_var": 6.278733781073242e-05 + }, + { + "episode": 16224, + "epoch": 0.8193939393939393, + "eps": 0, + "loss/policy_avg": 0.06992897391319275, + "loss/value_avg": 0.47664809226989746, + "lr": 8.76345189483222e-07, + "objective/entropy": -88.55311584472656, + "objective/kl": 30.213661193847656, + "objective/non_score_reward": -1.5106831789016724, + "objective/rlhf_reward": 13.747129440307617, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007972473395057023, + "policy/clipfrac_avg": 0.00806876365095377, + "policy/entropy_avg": 0.1417522430419922, + "step": 1014, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 51, + "val/ratio": 0.9906024932861328, + "val/ratio_var": 5.377350316848606e-05 + }, + { + "episode": 16240, + "epoch": 0.8202020202020202, + "eps": 0, + "loss/policy_avg": 0.11788127571344376, + "loss/value_avg": 0.528445839881897, + "lr": 8.688061284200266e-07, + "objective/entropy": -92.63037109375, + "objective/kl": 36.9803466796875, + "objective/non_score_reward": -1.84901762008667, + "objective/rlhf_reward": 14.065045356750488, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008528477046638727, + "policy/clipfrac_avg": 0.0033233293797820807, + "policy/entropy_avg": 0.17418956756591797, + "step": 1015, + "val/clipfrac_avg": 0.013305861502885818, + "val/num_eos_tokens": 387, + "val/ratio": 0.989635169506073, + "val/ratio_var": 4.9907346692634746e-05 + }, + { + "episode": 16256, + "epoch": 0.821010101010101, + "eps": 0, + "loss/policy_avg": -0.04949585720896721, + "loss/value_avg": 0.3462344706058502, + "lr": 8.61296549592941e-07, + "objective/entropy": -62.09973907470703, + "objective/kl": 44.77581024169922, + "objective/non_score_reward": -2.23879075050354, + "objective/rlhf_reward": 12.683084487915039, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.001046380726620555, + "policy/clipfrac_avg": 0.00542850699275732, + "policy/entropy_avg": 0.2170124053955078, + "step": 1016, + "val/clipfrac_avg": 0.03059275820851326, + "val/num_eos_tokens": 270, + "val/ratio": 0.9858708381652832, + "val/ratio_var": 6.539487367263064e-05 + }, + { + "episode": 16272, + "epoch": 0.8218181818181818, + "eps": 0, + "loss/policy_avg": 0.03657815232872963, + "loss/value_avg": 0.687595546245575, + "lr": 8.538165065940263e-07, + "objective/entropy": -65.8448486328125, + "objective/kl": 34.462608337402344, + "objective/non_score_reward": -1.7231305837631226, + "objective/rlhf_reward": 13.925307273864746, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000925581669434905, + "policy/clipfrac_avg": 0.00542820617556572, + "policy/entropy_avg": 0.19491100311279297, + "step": 1017, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 15, + "val/ratio": 0.9877764582633972, + "val/ratio_var": 6.733841291861609e-05 + }, + { + "episode": 16288, + "epoch": 0.8226262626262626, + "eps": 0, + "loss/policy_avg": -0.097564198076725, + "loss/value_avg": 0.5997859835624695, + "lr": 8.463660528045514e-07, + "objective/entropy": -21.665847778320312, + "objective/kl": 41.799583435058594, + "objective/non_score_reward": -2.0899791717529297, + "objective/rlhf_reward": 14.79283332824707, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010365031193941832, + "policy/clipfrac_avg": 0.00511179631575942, + "policy/entropy_avg": 0.20884323120117188, + "step": 1018, + "val/clipfrac_avg": 0.008413461968302727, + "val/num_eos_tokens": 123, + "val/ratio": 0.9861449003219604, + "val/ratio_var": 5.511847120942548e-05 + }, + { + "episode": 16304, + "epoch": 0.8234343434343434, + "eps": 0, + "loss/policy_avg": -0.06873838603496552, + "loss/value_avg": 0.2803807854652405, + "lr": 8.389452413946314e-07, + "objective/entropy": -106.4912109375, + "objective/kl": 40.70600891113281, + "objective/non_score_reward": -2.0353002548217773, + "objective/rlhf_reward": 13.745949745178223, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008316466701216996, + "policy/clipfrac_avg": 0.0057511827908456326, + "policy/entropy_avg": 0.1492767333984375, + "step": 1019, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 205, + "val/ratio": 0.9896730184555054, + "val/ratio_var": 4.522735616774298e-05 + }, + { + "episode": 16320, + "epoch": 0.8242424242424242, + "eps": 0, + "loss/policy_avg": 0.018638040870428085, + "loss/value_avg": 0.7235357761383057, + "lr": 8.315541253228332e-07, + "objective/entropy": -29.912065505981445, + "objective/kl": 32.72511291503906, + "objective/non_score_reward": -1.6362557411193848, + "objective/rlhf_reward": 13.707494735717773, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0009581202175468206, + "policy/clipfrac_avg": 0.0044068810530006886, + "policy/entropy_avg": 0.2027587890625, + "step": 1020, + "val/clipfrac_avg": 0.009765625, + "val/num_eos_tokens": 35, + "val/ratio": 0.9886679649353027, + "val/ratio_var": 7.175742939580232e-05 + }, + { + "episode": 16336, + "epoch": 0.825050505050505, + "eps": 0, + "loss/policy_avg": 0.15834374725818634, + "loss/value_avg": 0.9719556570053101, + "lr": 8.241927573358078e-07, + "objective/entropy": -17.822988510131836, + "objective/kl": 44.23792266845703, + "objective/non_score_reward": -2.2118961811065674, + "objective/rlhf_reward": 13.405291557312012, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.000986535451374948, + "policy/clipfrac_avg": 0.0043609049171209335, + "policy/entropy_avg": 0.19144058227539062, + "step": 1021, + "val/clipfrac_avg": 0.009959635324776173, + "val/num_eos_tokens": 44, + "val/ratio": 0.9864338040351868, + "val/ratio_var": 2.8990581995458342e-05 + }, + { + "episode": 16352, + "epoch": 0.8258585858585858, + "eps": 0, + "loss/policy_avg": 0.08962170779705048, + "loss/value_avg": 0.22370219230651855, + "lr": 8.168611899679013e-07, + "objective/entropy": -87.69083404541016, + "objective/kl": 41.61705017089844, + "objective/non_score_reward": -2.080852508544922, + "objective/rlhf_reward": 14.262897491455078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008991376380436122, + "policy/clipfrac_avg": 0.004512319341301918, + "policy/entropy_avg": 0.190032958984375, + "step": 1022, + "val/clipfrac_avg": 0.04466811940073967, + "val/num_eos_tokens": 17, + "val/ratio": 0.9874501824378967, + "val/ratio_var": 6.216822657734156e-05 + }, + { + "episode": 16368, + "epoch": 0.8266666666666667, + "eps": 0, + "loss/policy_avg": -0.6490851044654846, + "loss/value_avg": 0.6145610213279724, + "lr": 8.095594755407971e-07, + "objective/entropy": -103.04541778564453, + "objective/kl": 37.85612487792969, + "objective/non_score_reward": -1.8928062915802002, + "objective/rlhf_reward": 14.255630493164062, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009121837792918086, + "policy/clipfrac_avg": 0.004552682861685753, + "policy/entropy_avg": 0.17924374341964722, + "step": 1023, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 624, + "val/ratio": 0.9885929822921753, + "val/ratio_var": 8.515298395650461e-05 + }, + { + "episode": 16384, + "epoch": 0.8274747474747475, + "eps": 0, + "loss/policy_avg": 0.09050679206848145, + "loss/value_avg": 1.2950150966644287, + "lr": 8.022876661631268e-07, + "objective/entropy": -46.773006439208984, + "objective/kl": 33.627471923828125, + "objective/non_score_reward": -1.6813737154006958, + "objective/rlhf_reward": 14.521751403808594, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009958728915080428, + "policy/clipfrac_avg": 0.0035771888215094805, + "policy/entropy_avg": 0.17354488372802734, + "step": 1024, + "val/clipfrac_avg": 0.0034391595982015133, + "val/num_eos_tokens": 5, + "val/ratio": 0.9874868392944336, + "val/ratio_var": 7.542264211224392e-05 + }, + { + "episode": 16400, + "epoch": 0.8282828282828283, + "eps": 0, + "loss/policy_avg": 0.09637929499149323, + "loss/value_avg": 0.3471735417842865, + "lr": 7.950458137301082e-07, + "objective/entropy": -118.60977172851562, + "objective/kl": 40.922630310058594, + "objective/non_score_reward": -2.0461313724517822, + "objective/rlhf_reward": 14.555431365966797, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007377723813988268, + "policy/clipfrac_avg": 0.004692882765084505, + "policy/entropy_avg": 0.1540050506591797, + "step": 1025, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 445, + "val/ratio": 0.9898372888565063, + "val/ratio_var": 4.050094139529392e-05 + }, + { + "episode": 16416, + "epoch": 0.8290909090909091, + "eps": 0, + "loss/policy_avg": 0.003938224166631699, + "loss/value_avg": 0.12186452746391296, + "lr": 7.878339699231702e-07, + "objective/entropy": 5.97334623336792, + "objective/kl": 47.65478515625, + "objective/non_score_reward": -2.382739543914795, + "objective/rlhf_reward": 15.882884979248047, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011957846581935883, + "policy/clipfrac_avg": 0.0038270659279078245, + "policy/entropy_avg": 0.24349212646484375, + "step": 1026, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 63, + "val/ratio": 0.9836880564689636, + "val/ratio_var": 3.132951314910315e-05 + }, + { + "episode": 16432, + "epoch": 0.8298989898989899, + "eps": 0, + "loss/policy_avg": -0.008172400295734406, + "loss/value_avg": 0.40194565057754517, + "lr": 7.806521862095834e-07, + "objective/entropy": -145.87228393554688, + "objective/kl": 43.73109817504883, + "objective/non_score_reward": -2.1865549087524414, + "objective/rlhf_reward": 12.707976341247559, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000986664555966854, + "policy/clipfrac_avg": 0.005425687879323959, + "policy/entropy_avg": 0.21741676330566406, + "step": 1027, + "val/clipfrac_avg": 0.00168609619140625, + "val/num_eos_tokens": 2002, + "val/ratio": 0.9879361987113953, + "val/ratio_var": 7.765040209051222e-05 + }, + { + "episode": 16448, + "epoch": 0.8307070707070707, + "eps": 0, + "loss/policy_avg": -0.040230363607406616, + "loss/value_avg": 0.2879355549812317, + "lr": 7.735005138421003e-07, + "objective/entropy": -39.62914276123047, + "objective/kl": 30.83833122253418, + "objective/non_score_reward": -1.5419166088104248, + "objective/rlhf_reward": 13.919020652770996, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008982427534647286, + "policy/clipfrac_avg": 0.004154416732490063, + "policy/entropy_avg": 0.1513214111328125, + "step": 1028, + "val/clipfrac_avg": 0.0003633720916695893, + "val/num_eos_tokens": 42, + "val/ratio": 0.9900394678115845, + "val/ratio_var": 4.5976830733707175e-05 + }, + { + "episode": 16464, + "epoch": 0.8315151515151515, + "eps": 0, + "loss/policy_avg": -0.17712682485580444, + "loss/value_avg": 0.41484764218330383, + "lr": 7.663790038585794e-07, + "objective/entropy": -91.81086730957031, + "objective/kl": 39.669403076171875, + "objective/non_score_reward": -1.9834702014923096, + "objective/rlhf_reward": 14.73528003692627, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009472984238527715, + "policy/clipfrac_avg": 0.0033632363192737103, + "policy/entropy_avg": 0.1894826889038086, + "step": 1029, + "val/clipfrac_avg": 0.015109826810657978, + "val/num_eos_tokens": 637, + "val/ratio": 0.9873372316360474, + "val/ratio_var": 6.729063170496374e-05 + }, + { + "episode": 16480, + "epoch": 0.8323232323232324, + "eps": 0, + "loss/policy_avg": 0.03287976607680321, + "loss/value_avg": 0.3657122254371643, + "lr": 7.592877070816268e-07, + "objective/entropy": -81.92291259765625, + "objective/kl": 40.63905334472656, + "objective/non_score_reward": -2.0319526195526123, + "objective/rlhf_reward": 14.960234642028809, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009109035599976778, + "policy/clipfrac_avg": 0.003938743844628334, + "policy/entropy_avg": 0.17749857902526855, + "step": 1030, + "val/clipfrac_avg": 0.005359000526368618, + "val/num_eos_tokens": 56, + "val/ratio": 0.9891733527183533, + "val/ratio_var": 5.9037287428509444e-05 + }, + { + "episode": 16496, + "epoch": 0.8331313131313132, + "eps": 0, + "loss/policy_avg": 0.054874081164598465, + "loss/value_avg": 0.28462013602256775, + "lr": 7.522266741182305e-07, + "objective/entropy": -79.7171630859375, + "objective/kl": 43.41844177246094, + "objective/non_score_reward": -2.170922040939331, + "objective/rlhf_reward": 13.88376522064209, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009897679556161165, + "policy/clipfrac_avg": 0.00460906233638525, + "policy/entropy_avg": 0.18915176391601562, + "step": 1031, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 413, + "val/ratio": 0.987737238407135, + "val/ratio_var": 6.536820001201704e-05 + }, + { + "episode": 16512, + "epoch": 0.833939393939394, + "eps": 0, + "loss/policy_avg": 0.42096102237701416, + "loss/value_avg": 0.7095996141433716, + "lr": 7.451959553594051e-07, + "objective/entropy": -75.63118743896484, + "objective/kl": 32.546966552734375, + "objective/non_score_reward": -1.6273484230041504, + "objective/rlhf_reward": 12.075777053833008, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.000955111114308238, + "policy/clipfrac_avg": 0.005775670520961285, + "policy/entropy_avg": 0.16732215881347656, + "step": 1032, + "val/clipfrac_avg": 0.017921777442097664, + "val/num_eos_tokens": 145, + "val/ratio": 0.989020586013794, + "val/ratio_var": 4.018365507363342e-05 + }, + { + "episode": 16528, + "epoch": 0.8347474747474748, + "eps": 0, + "loss/policy_avg": 0.017501141875982285, + "loss/value_avg": 0.4045110046863556, + "lr": 7.381956009798253e-07, + "objective/entropy": -53.61748123168945, + "objective/kl": 38.599403381347656, + "objective/non_score_reward": -1.929970145225525, + "objective/rlhf_reward": 12.640341758728027, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008390340954065323, + "policy/clipfrac_avg": 0.0041229561902582645, + "policy/entropy_avg": 0.18261051177978516, + "step": 1033, + "val/clipfrac_avg": 0.020542385056614876, + "val/num_eos_tokens": 139, + "val/ratio": 0.988516092300415, + "val/ratio_var": 9.491335367783904e-05 + }, + { + "episode": 16544, + "epoch": 0.8355555555555556, + "eps": 0, + "loss/policy_avg": -0.03480757772922516, + "loss/value_avg": 1.401247501373291, + "lr": 7.312256609374708e-07, + "objective/entropy": -103.81524658203125, + "objective/kl": 36.218544006347656, + "objective/non_score_reward": -1.810927152633667, + "objective/rlhf_reward": 14.07969856262207, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007683830335736275, + "policy/clipfrac_avg": 0.003070900682359934, + "policy/entropy_avg": 0.14527511596679688, + "step": 1034, + "val/clipfrac_avg": 0.03183094039559364, + "val/num_eos_tokens": 65, + "val/ratio": 0.9899626970291138, + "val/ratio_var": 6.579139153473079e-05 + }, + { + "episode": 16560, + "epoch": 0.8363636363636363, + "eps": 0, + "loss/policy_avg": -0.28031888604164124, + "loss/value_avg": 1.1235523223876953, + "lr": 7.242861849732696e-07, + "objective/entropy": -59.279266357421875, + "objective/kl": 38.71485900878906, + "objective/non_score_reward": -1.935742974281311, + "objective/rlhf_reward": 14.07206916809082, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007580246892757714, + "policy/clipfrac_avg": 0.003035132307559252, + "policy/entropy_avg": 0.14683592319488525, + "step": 1035, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 109, + "val/ratio": 0.9899235963821411, + "val/ratio_var": 5.906899241381325e-05 + }, + { + "episode": 16576, + "epoch": 0.8371717171717171, + "eps": 0, + "loss/policy_avg": -0.14042839407920837, + "loss/value_avg": 1.1183263063430786, + "lr": 7.173772226107434e-07, + "objective/entropy": -64.92012023925781, + "objective/kl": 38.85520553588867, + "objective/non_score_reward": -1.9427603483200073, + "objective/rlhf_reward": 14.400989532470703, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009576823795214295, + "policy/clipfrac_avg": 0.004880891181528568, + "policy/entropy_avg": 0.19240808486938477, + "step": 1036, + "val/clipfrac_avg": 0.012853971682488918, + "val/num_eos_tokens": 155, + "val/ratio": 0.987326979637146, + "val/ratio_var": 9.263003448722884e-05 + }, + { + "episode": 16592, + "epoch": 0.8379797979797979, + "eps": 0, + "loss/policy_avg": -0.03311082720756531, + "loss/value_avg": 0.3677228093147278, + "lr": 7.104988231556543e-07, + "objective/entropy": -48.31360626220703, + "objective/kl": 36.47294616699219, + "objective/non_score_reward": -1.8236472606658936, + "objective/rlhf_reward": 14.137290000915527, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008876229985617101, + "policy/clipfrac_avg": 0.002901179250329733, + "policy/entropy_avg": 0.20080184936523438, + "step": 1037, + "val/clipfrac_avg": 0.002261103130877018, + "val/num_eos_tokens": 20, + "val/ratio": 0.9893977642059326, + "val/ratio_var": 5.7600576838012785e-05 + }, + { + "episode": 16608, + "epoch": 0.8387878787878787, + "eps": 0, + "loss/policy_avg": -0.10436652600765228, + "loss/value_avg": 0.43171289563179016, + "lr": 7.036510356956494e-07, + "objective/entropy": -66.36492919921875, + "objective/kl": 41.474632263183594, + "objective/non_score_reward": -2.0737314224243164, + "objective/rlhf_reward": 15.168456077575684, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0009756431682035327, + "policy/clipfrac_avg": 0.005543471779674292, + "policy/entropy_avg": 0.188018798828125, + "step": 1038, + "val/clipfrac_avg": 0.01755008101463318, + "val/num_eos_tokens": 3, + "val/ratio": 0.9874497652053833, + "val/ratio_var": 5.146233888808638e-05 + }, + { + "episode": 16624, + "epoch": 0.8395959595959596, + "eps": 0, + "loss/policy_avg": -0.23364350199699402, + "loss/value_avg": 0.3087168335914612, + "lr": 6.968339090999188e-07, + "objective/entropy": -82.42080688476562, + "objective/kl": 33.46930694580078, + "objective/non_score_reward": -1.6734652519226074, + "objective/rlhf_reward": 16.849971771240234, + "objective/scores": 18.5, + "policy/approxkl_avg": 0.0009166139643639326, + "policy/clipfrac_avg": 0.0028331982903182507, + "policy/entropy_avg": 0.18764019012451172, + "step": 1039, + "val/clipfrac_avg": 0.022823384031653404, + "val/num_eos_tokens": 150, + "val/ratio": 0.9876014590263367, + "val/ratio_var": 7.868735701777041e-05 + }, + { + "episode": 16640, + "epoch": 0.8404040404040404, + "eps": 0, + "loss/policy_avg": -0.24348971247673035, + "loss/value_avg": 0.46729177236557007, + "lr": 6.900474920188383e-07, + "objective/entropy": -98.47803497314453, + "objective/kl": 31.56005096435547, + "objective/non_score_reward": -1.5780024528503418, + "objective/rlhf_reward": 12.9376220703125, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0007581596728414297, + "policy/clipfrac_avg": 0.0029252313543111086, + "policy/entropy_avg": 0.14857721328735352, + "step": 1040, + "val/clipfrac_avg": 0.018430940806865692, + "val/num_eos_tokens": 64, + "val/ratio": 0.9900062084197998, + "val/ratio_var": 7.637133239768445e-05 + }, + { + "episode": 16656, + "epoch": 0.8412121212121212, + "eps": 0, + "loss/policy_avg": -0.3006308078765869, + "loss/value_avg": 0.9436385035514832, + "lr": 6.832918328836247e-07, + "objective/entropy": -67.75382995605469, + "objective/kl": 32.892269134521484, + "objective/non_score_reward": -1.64461350440979, + "objective/rlhf_reward": 14.644449234008789, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009258019272238016, + "policy/clipfrac_avg": 0.004761096090078354, + "policy/entropy_avg": 0.1688389778137207, + "step": 1041, + "val/clipfrac_avg": 0.018809743225574493, + "val/num_eos_tokens": 357, + "val/ratio": 0.9893865585327148, + "val/ratio_var": 6.368513277266175e-05 + }, + { + "episode": 16672, + "epoch": 0.842020202020202, + "eps": 0, + "loss/policy_avg": -0.06437136977910995, + "loss/value_avg": 0.4288250207901001, + "lr": 6.765669799059915e-07, + "objective/entropy": -74.0018310546875, + "objective/kl": 35.5787467956543, + "objective/non_score_reward": -1.7789373397827148, + "objective/rlhf_reward": 13.291374206542969, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008827393758110702, + "policy/clipfrac_avg": 0.004510640166699886, + "policy/entropy_avg": 0.174835205078125, + "step": 1042, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 88, + "val/ratio": 0.9889439344406128, + "val/ratio_var": 5.196507117943838e-05 + }, + { + "episode": 16688, + "epoch": 0.8428282828282828, + "eps": 0, + "loss/policy_avg": 0.08304592221975327, + "loss/value_avg": 0.627299427986145, + "lr": 6.698729810778065e-07, + "objective/entropy": -94.1718521118164, + "objective/kl": 36.0411491394043, + "objective/non_score_reward": -1.8020575046539307, + "objective/rlhf_reward": 14.455755233764648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007288270280696452, + "policy/clipfrac_avg": 0.003802281804382801, + "policy/entropy_avg": 0.14253592491149902, + "step": 1043, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 492, + "val/ratio": 0.9900709390640259, + "val/ratio_var": 3.72955619241111e-05 + }, + { + "episode": 16704, + "epoch": 0.8436363636363636, + "eps": 0, + "loss/policy_avg": 0.289115309715271, + "loss/value_avg": 0.6000362038612366, + "lr": 6.632098841707458e-07, + "objective/entropy": -44.56480407714844, + "objective/kl": 37.80200958251953, + "objective/non_score_reward": -1.8901004791259766, + "objective/rlhf_reward": 13.203649520874023, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0012037131236866117, + "policy/clipfrac_avg": 0.005290274508297443, + "policy/entropy_avg": 0.21977996826171875, + "step": 1044, + "val/clipfrac_avg": 0.0035122863482683897, + "val/num_eos_tokens": 378, + "val/ratio": 0.9847176671028137, + "val/ratio_var": 9.624927770346403e-05 + }, + { + "episode": 16720, + "epoch": 0.8444444444444444, + "eps": 0, + "loss/policy_avg": 0.07545819133520126, + "loss/value_avg": 0.46948039531707764, + "lr": 6.565777367359516e-07, + "objective/entropy": -68.83721923828125, + "objective/kl": 39.69865798950195, + "objective/non_score_reward": -1.9849328994750977, + "objective/rlhf_reward": 13.468192100524902, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008786184480413795, + "policy/clipfrac_avg": 0.004983194172382355, + "policy/entropy_avg": 0.17647552490234375, + "step": 1045, + "val/clipfrac_avg": 0.00048828125, + "val/num_eos_tokens": 99, + "val/ratio": 0.9880126714706421, + "val/ratio_var": 4.486176840146072e-05 + }, + { + "episode": 16736, + "epoch": 0.8452525252525253, + "eps": 0, + "loss/policy_avg": 0.18258720636367798, + "loss/value_avg": 0.33022794127464294, + "lr": 6.499765861036994e-07, + "objective/entropy": -133.97775268554688, + "objective/kl": 39.10038375854492, + "objective/non_score_reward": -1.9550191164016724, + "objective/rlhf_reward": 13.568418502807617, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007223144639283419, + "policy/clipfrac_avg": 0.0023505911231040955, + "policy/entropy_avg": 0.1742839813232422, + "step": 1046, + "val/clipfrac_avg": 0.001006610575132072, + "val/num_eos_tokens": 767, + "val/ratio": 0.9903188943862915, + "val/ratio_var": 4.2478550312807783e-05 + }, + { + "episode": 16752, + "epoch": 0.8460606060606061, + "eps": 0, + "loss/policy_avg": 0.004041165113449097, + "loss/value_avg": 0.2533167600631714, + "lr": 6.43406479383053e-07, + "objective/entropy": -11.881628036499023, + "objective/kl": 40.72428894042969, + "objective/non_score_reward": -2.036214590072632, + "objective/rlhf_reward": 15.112222671508789, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0010564490221440792, + "policy/clipfrac_avg": 0.007933112792670727, + "policy/entropy_avg": 0.20357513427734375, + "step": 1047, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.9868043661117554, + "val/ratio_var": 6.374649819917977e-05 + }, + { + "episode": 16768, + "epoch": 0.8468686868686869, + "eps": 0, + "loss/policy_avg": 0.0763624757528305, + "loss/value_avg": 0.4388294219970703, + "lr": 6.368674634615357e-07, + "objective/entropy": -45.377681732177734, + "objective/kl": 34.73613739013672, + "objective/non_score_reward": -1.736806869506836, + "objective/rlhf_reward": 14.192880630493164, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009812908247113228, + "policy/clipfrac_avg": 0.0049714683555066586, + "policy/entropy_avg": 0.18614768981933594, + "step": 1048, + "val/clipfrac_avg": 0.0006127451197244227, + "val/num_eos_tokens": 351, + "val/ratio": 0.98753422498703, + "val/ratio_var": 4.261466165189631e-05 + }, + { + "episode": 16784, + "epoch": 0.8476767676767677, + "eps": 0, + "loss/policy_avg": -0.12992079555988312, + "loss/value_avg": 0.939970850944519, + "lr": 6.303595850047844e-07, + "objective/entropy": -80.03951263427734, + "objective/kl": 42.18169021606445, + "objective/non_score_reward": -2.1090846061706543, + "objective/rlhf_reward": 13.812790870666504, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009452086524106562, + "policy/clipfrac_avg": 0.004814188927412033, + "policy/entropy_avg": 0.1846628189086914, + "step": 1049, + "val/clipfrac_avg": 0.0003124999930150807, + "val/num_eos_tokens": 193, + "val/ratio": 0.988731324672699, + "val/ratio_var": 6.252430466702208e-05 + }, + { + "episode": 16800, + "epoch": 0.8484848484848485, + "eps": 0, + "loss/policy_avg": -0.12597322463989258, + "loss/value_avg": 0.5964376926422119, + "lr": 6.238828904562316e-07, + "objective/entropy": -56.887001037597656, + "objective/kl": 37.25836944580078, + "objective/non_score_reward": -1.8629186153411865, + "objective/rlhf_reward": 14.301143646240234, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008069226169027388, + "policy/clipfrac_avg": 0.0021998940501362085, + "policy/entropy_avg": 0.17313003540039062, + "step": 1050, + "val/clipfrac_avg": 0.00014467592700384557, + "val/num_eos_tokens": 69, + "val/ratio": 0.9889218807220459, + "val/ratio_var": 4.452227949514054e-05 + }, + { + "episode": 16816, + "epoch": 0.8492929292929293, + "eps": 0, + "loss/policy_avg": -0.284543514251709, + "loss/value_avg": 0.4945632815361023, + "lr": 6.174374260367611e-07, + "objective/entropy": -162.38363647460938, + "objective/kl": 36.81440734863281, + "objective/non_score_reward": -1.8407204151153564, + "objective/rlhf_reward": 13.549904823303223, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0006882078596390784, + "policy/clipfrac_avg": 0.0029306234791874886, + "policy/entropy_avg": 0.13589084148406982, + "step": 1051, + "val/clipfrac_avg": 0.02376413717865944, + "val/num_eos_tokens": 810, + "val/ratio": 0.9912172555923462, + "val/ratio_var": 4.5012933696853e-05 + }, + { + "episode": 16832, + "epoch": 0.8501010101010101, + "eps": 0, + "loss/policy_avg": -0.011637904681265354, + "loss/value_avg": 0.556053638458252, + "lr": 6.110232377443864e-07, + "objective/entropy": -106.74056243896484, + "objective/kl": 35.95270538330078, + "objective/non_score_reward": -1.7976351976394653, + "objective/rlhf_reward": 16.46017837524414, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0009089771774597466, + "policy/clipfrac_avg": 0.005288749001920223, + "policy/entropy_avg": 0.18283462524414062, + "step": 1052, + "val/clipfrac_avg": 0.0026041667442768812, + "val/num_eos_tokens": 104, + "val/ratio": 0.9882605075836182, + "val/ratio_var": 7.140174420783296e-05 + }, + { + "episode": 16848, + "epoch": 0.850909090909091, + "eps": 0, + "loss/policy_avg": -0.09856721758842468, + "loss/value_avg": 0.43345025181770325, + "lr": 6.04640371353914e-07, + "objective/entropy": -75.6595458984375, + "objective/kl": 40.826602935791016, + "objective/non_score_reward": -2.041330337524414, + "objective/rlhf_reward": 13.857107162475586, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009147390373982489, + "policy/clipfrac_avg": 0.0046502938494086266, + "policy/entropy_avg": 0.21721649169921875, + "step": 1053, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 61, + "val/ratio": 0.9873005151748657, + "val/ratio_var": 7.706157339271158e-05 + }, + { + "episode": 16864, + "epoch": 0.8517171717171718, + "eps": 0, + "loss/policy_avg": 0.051140159368515015, + "loss/value_avg": 0.46211516857147217, + "lr": 5.98288872416628e-07, + "objective/entropy": -99.58453369140625, + "objective/kl": 38.763389587402344, + "objective/non_score_reward": -1.938169240951538, + "objective/rlhf_reward": 15.054018020629883, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008463884005323052, + "policy/clipfrac_avg": 0.0038104294799268246, + "policy/entropy_avg": 0.16744399070739746, + "step": 1054, + "val/clipfrac_avg": 0.0019132653251290321, + "val/num_eos_tokens": 520, + "val/ratio": 0.989348292350769, + "val/ratio_var": 8.285223157145083e-05 + }, + { + "episode": 16880, + "epoch": 0.8525252525252526, + "eps": 0, + "loss/policy_avg": -0.03345664590597153, + "loss/value_avg": 0.41552969813346863, + "lr": 5.919687862599549e-07, + "objective/entropy": -79.0547103881836, + "objective/kl": 37.5797004699707, + "objective/non_score_reward": -1.878985047340393, + "objective/rlhf_reward": 14.566328048706055, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009661274962127209, + "policy/clipfrac_avg": 0.00524417357519269, + "policy/entropy_avg": 0.186629980802536, + "step": 1055, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 305, + "val/ratio": 0.9881995916366577, + "val/ratio_var": 7.692445797147229e-05 + }, + { + "episode": 16896, + "epoch": 0.8533333333333334, + "eps": 0, + "loss/policy_avg": 0.1846567541360855, + "loss/value_avg": 0.5820722579956055, + "lr": 5.856801579871457e-07, + "objective/entropy": -91.50765991210938, + "objective/kl": 38.447975158691406, + "objective/non_score_reward": -1.9223988056182861, + "objective/rlhf_reward": 12.522913932800293, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0007886831881478429, + "policy/clipfrac_avg": 0.00426090694963932, + "policy/entropy_avg": 0.15495681762695312, + "step": 1056, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 137, + "val/ratio": 0.9909533262252808, + "val/ratio_var": 5.192918251850642e-05 + }, + { + "episode": 16912, + "epoch": 0.8541414141414141, + "eps": 0, + "loss/policy_avg": -0.014410331845283508, + "loss/value_avg": 0.45579588413238525, + "lr": 5.794230324769518e-07, + "objective/entropy": -45.45872497558594, + "objective/kl": 41.44641876220703, + "objective/non_score_reward": -2.0723209381103516, + "objective/rlhf_reward": 13.966741561889648, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009499872685410082, + "policy/clipfrac_avg": 0.007878290489315987, + "policy/entropy_avg": 0.18343734741210938, + "step": 1057, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 399, + "val/ratio": 0.9887840747833252, + "val/ratio_var": 6.99822703609243e-05 + }, + { + "episode": 16928, + "epoch": 0.8549494949494949, + "eps": 0, + "loss/policy_avg": -0.05935902148485184, + "loss/value_avg": 0.36739227175712585, + "lr": 5.731974543833041e-07, + "objective/entropy": -55.36639404296875, + "objective/kl": 38.324073791503906, + "objective/non_score_reward": -1.916203498840332, + "objective/rlhf_reward": 12.029109001159668, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009155046427622437, + "policy/clipfrac_avg": 0.003906010650098324, + "policy/entropy_avg": 0.17214488983154297, + "step": 1058, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891675114631653, + "val/ratio_var": 5.269415851216763e-05 + }, + { + "episode": 16944, + "epoch": 0.8557575757575757, + "eps": 0, + "loss/policy_avg": 0.12509797513484955, + "loss/value_avg": 0.5901766419410706, + "lr": 5.670034681349995e-07, + "objective/entropy": -77.39466857910156, + "objective/kl": 39.27944564819336, + "objective/non_score_reward": -1.9639724493026733, + "objective/rlhf_reward": 12.512590408325195, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0011488801101222634, + "policy/clipfrac_avg": 0.005929230712354183, + "policy/entropy_avg": 0.22491836547851562, + "step": 1059, + "val/clipfrac_avg": 0.01009130198508501, + "val/num_eos_tokens": 274, + "val/ratio": 0.9847910404205322, + "val/ratio_var": 8.84500186657533e-05 + }, + { + "episode": 16960, + "epoch": 0.8565656565656565, + "eps": 0, + "loss/policy_avg": 0.027433840557932854, + "loss/value_avg": 0.2936461567878723, + "lr": 5.608411179353767e-07, + "objective/entropy": -68.70003509521484, + "objective/kl": 34.862098693847656, + "objective/non_score_reward": -1.7431049346923828, + "objective/rlhf_reward": 15.538145065307617, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010132534662261605, + "policy/clipfrac_avg": 0.00747870746999979, + "policy/entropy_avg": 0.19480419158935547, + "step": 1060, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 6, + "val/ratio": 0.9880098700523376, + "val/ratio_var": 6.574365397682413e-05 + }, + { + "episode": 16976, + "epoch": 0.8573737373737373, + "eps": 0, + "loss/policy_avg": 0.22168952226638794, + "loss/value_avg": 0.6544172763824463, + "lr": 5.547104477620041e-07, + "objective/entropy": -101.6231918334961, + "objective/kl": 36.893959045410156, + "objective/non_score_reward": -1.844698190689087, + "objective/rlhf_reward": 13.850614547729492, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008218783186748624, + "policy/clipfrac_avg": 0.005969468504190445, + "policy/entropy_avg": 0.15513205528259277, + "step": 1061, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 465, + "val/ratio": 0.9892902374267578, + "val/ratio_var": 6.282331742113456e-05 + }, + { + "episode": 16992, + "epoch": 0.8581818181818182, + "eps": 0, + "loss/policy_avg": -0.23859268426895142, + "loss/value_avg": 0.5853025913238525, + "lr": 5.486115013663668e-07, + "objective/entropy": -112.72584533691406, + "objective/kl": 33.110145568847656, + "objective/non_score_reward": -1.6555070877075195, + "objective/rlhf_reward": 15.28980541229248, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0006752893095836043, + "policy/clipfrac_avg": 0.0032210415229201317, + "policy/entropy_avg": 0.12448978424072266, + "step": 1062, + "val/clipfrac_avg": 4.234417428961024e-05, + "val/num_eos_tokens": 451, + "val/ratio": 0.9915745258331299, + "val/ratio_var": 5.202127795200795e-05 + }, + { + "episode": 17008, + "epoch": 0.858989898989899, + "eps": 0, + "loss/policy_avg": -0.1410359889268875, + "loss/value_avg": 0.22414374351501465, + "lr": 5.425443222735527e-07, + "objective/entropy": -49.841190338134766, + "objective/kl": 38.634700775146484, + "objective/non_score_reward": -1.9317350387573242, + "objective/rlhf_reward": 16.25576400756836, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011730990372598171, + "policy/clipfrac_avg": 0.008051658049225807, + "policy/entropy_avg": 0.22179222106933594, + "step": 1063, + "val/clipfrac_avg": 0.009043844416737556, + "val/num_eos_tokens": 125, + "val/ratio": 0.9854654669761658, + "val/ratio_var": 7.81394264777191e-05 + }, + { + "episode": 17024, + "epoch": 0.8597979797979798, + "eps": 0, + "loss/policy_avg": -0.060928136110305786, + "loss/value_avg": 0.45962244272232056, + "lr": 5.365089537819435e-07, + "objective/entropy": -68.24198913574219, + "objective/kl": 37.95829772949219, + "objective/non_score_reward": -1.8979148864746094, + "objective/rlhf_reward": 15.38333511352539, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008637227001599967, + "policy/clipfrac_avg": 0.006955782417207956, + "policy/entropy_avg": 0.1758570671081543, + "step": 1064, + "val/clipfrac_avg": 0.027727147564291954, + "val/num_eos_tokens": 423, + "val/ratio": 0.9886413812637329, + "val/ratio_var": 8.117071411106735e-05 + }, + { + "episode": 17040, + "epoch": 0.8606060606060606, + "eps": 0, + "loss/policy_avg": -0.024640943855047226, + "loss/value_avg": 0.736940324306488, + "lr": 5.305054389629022e-07, + "objective/entropy": -66.94715881347656, + "objective/kl": 41.044742584228516, + "objective/non_score_reward": -2.052237033843994, + "objective/rlhf_reward": 13.018075942993164, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009623835212551057, + "policy/clipfrac_avg": 0.0037963923532515764, + "policy/entropy_avg": 0.1932220458984375, + "step": 1065, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 86, + "val/ratio": 0.9877476692199707, + "val/ratio_var": 5.7415742048760876e-05 + }, + { + "episode": 17056, + "epoch": 0.8614141414141414, + "eps": 0, + "loss/policy_avg": 0.11599922180175781, + "loss/value_avg": 1.5807065963745117, + "lr": 5.245338206604733e-07, + "objective/entropy": -115.46347045898438, + "objective/kl": 40.31273651123047, + "objective/non_score_reward": -2.015636682510376, + "objective/rlhf_reward": 13.367176055908203, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008412263705395162, + "policy/clipfrac_avg": 0.01834527961909771, + "policy/entropy_avg": 0.17316818237304688, + "step": 1066, + "val/clipfrac_avg": 1.0643732821336016e-05, + "val/num_eos_tokens": 71, + "val/ratio": 0.9887286424636841, + "val/ratio_var": 4.431970592122525e-05 + }, + { + "episode": 17072, + "epoch": 0.8622222222222222, + "eps": 0, + "loss/policy_avg": -0.11949857324361801, + "loss/value_avg": 0.5354464650154114, + "lr": 5.185941414910673e-07, + "objective/entropy": -67.14189147949219, + "objective/kl": 39.36028289794922, + "objective/non_score_reward": -1.9680140018463135, + "objective/rlhf_reward": 15.414798736572266, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0008963524596765637, + "policy/clipfrac_avg": 0.00728583661839366, + "policy/entropy_avg": 0.17730861902236938, + "step": 1067, + "val/clipfrac_avg": 0.0072115384973585606, + "val/num_eos_tokens": 286, + "val/ratio": 0.9881836175918579, + "val/ratio_var": 3.5629829653771594e-05 + }, + { + "episode": 17088, + "epoch": 0.863030303030303, + "eps": 0, + "loss/policy_avg": 0.10199490189552307, + "loss/value_avg": 0.35868531465530396, + "lr": 5.126864438431628e-07, + "objective/entropy": -115.96305847167969, + "objective/kl": 37.194644927978516, + "objective/non_score_reward": -1.8597323894500732, + "objective/rlhf_reward": 12.882455825805664, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000633776537142694, + "policy/clipfrac_avg": 0.004217084962874651, + "policy/entropy_avg": 0.13162803649902344, + "step": 1068, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 156, + "val/ratio": 0.9914271235466003, + "val/ratio_var": 5.447517105494626e-05 + }, + { + "episode": 17104, + "epoch": 0.8638383838383838, + "eps": 0, + "loss/policy_avg": 0.09657607972621918, + "loss/value_avg": 0.33049535751342773, + "lr": 5.06810769877002e-07, + "objective/entropy": -27.455425262451172, + "objective/kl": 30.918790817260742, + "objective/non_score_reward": -1.545939564704895, + "objective/rlhf_reward": 13.969684600830078, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000932033930439502, + "policy/clipfrac_avg": 0.0023164437152445316, + "policy/entropy_avg": 0.2050342559814453, + "step": 1069, + "val/clipfrac_avg": 0.0017710273386910558, + "val/num_eos_tokens": 117, + "val/ratio": 0.9869778752326965, + "val/ratio_var": 7.318282587220892e-05 + }, + { + "episode": 17120, + "epoch": 0.8646464646464647, + "eps": 0, + "loss/policy_avg": -0.07673387229442596, + "loss/value_avg": 0.6429687738418579, + "lr": 5.009671615242928e-07, + "objective/entropy": -41.616695404052734, + "objective/kl": 42.76515197753906, + "objective/non_score_reward": -2.1382575035095215, + "objective/rlhf_reward": 13.17424201965332, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009901263983920217, + "policy/clipfrac_avg": 0.003959751222282648, + "policy/entropy_avg": 0.1980295181274414, + "step": 1070, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 199, + "val/ratio": 0.987491250038147, + "val/ratio_var": 4.666786844609305e-05 + }, + { + "episode": 17136, + "epoch": 0.8654545454545455, + "eps": 0, + "loss/policy_avg": -0.03734193369746208, + "loss/value_avg": 0.7440835237503052, + "lr": 4.951556604879049e-07, + "objective/entropy": -53.619712829589844, + "objective/kl": 39.59283447265625, + "objective/non_score_reward": -1.9796417951583862, + "objective/rlhf_reward": 13.887545585632324, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010703656589612365, + "policy/clipfrac_avg": 0.0033148752991110086, + "policy/entropy_avg": 0.19559097290039062, + "step": 1071, + "val/clipfrac_avg": 0.00032552084303461015, + "val/num_eos_tokens": 134, + "val/ratio": 0.9871270060539246, + "val/ratio_var": 6.861035944893956e-05 + }, + { + "episode": 17152, + "epoch": 0.8662626262626263, + "eps": 0, + "loss/policy_avg": 0.05410761386156082, + "loss/value_avg": 0.6214333772659302, + "lr": 4.893763082415736e-07, + "objective/entropy": -32.19466781616211, + "objective/kl": 33.41657638549805, + "objective/non_score_reward": -1.6708288192749023, + "objective/rlhf_reward": 12.516671180725098, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010530936997383833, + "policy/clipfrac_avg": 0.005051491782069206, + "policy/entropy_avg": 0.18621063232421875, + "step": 1072, + "val/clipfrac_avg": 0.006988893263041973, + "val/num_eos_tokens": 41, + "val/ratio": 0.9861816167831421, + "val/ratio_var": 5.813246752950363e-05 + }, + { + "episode": 17168, + "epoch": 0.8670707070707071, + "eps": 0, + "loss/policy_avg": -0.13285210728645325, + "loss/value_avg": 0.37682080268859863, + "lr": 4.836291460296066e-07, + "objective/entropy": -73.75119018554688, + "objective/kl": 44.275638580322266, + "objective/non_score_reward": -2.2137818336486816, + "objective/rlhf_reward": 13.872156143188477, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009736482752487063, + "policy/clipfrac_avg": 0.002118697389960289, + "policy/entropy_avg": 0.1989288330078125, + "step": 1073, + "val/clipfrac_avg": 0.003361742477864027, + "val/num_eos_tokens": 178, + "val/ratio": 0.9864859580993652, + "val/ratio_var": 7.720785652054474e-05 + }, + { + "episode": 17184, + "epoch": 0.8678787878787879, + "eps": 0, + "loss/policy_avg": 0.032950110733509064, + "loss/value_avg": 0.5699096918106079, + "lr": 4.779142148665855e-07, + "objective/entropy": -17.03137969970703, + "objective/kl": 29.793800354003906, + "objective/non_score_reward": -1.4896900653839111, + "objective/rlhf_reward": 15.447810173034668, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.000868711736984551, + "policy/clipfrac_avg": 0.0015887501649558544, + "policy/entropy_avg": 0.17066001892089844, + "step": 1074, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 84, + "val/ratio": 0.9895613193511963, + "val/ratio_var": 7.112583261914551e-05 + }, + { + "episode": 17200, + "epoch": 0.8686868686868687, + "eps": 0, + "loss/policy_avg": -0.11098457127809525, + "loss/value_avg": 0.5394589304924011, + "lr": 4.722315555370793e-07, + "objective/entropy": -46.34983825683594, + "objective/kl": 39.953025817871094, + "objective/non_score_reward": -1.997651219367981, + "objective/rlhf_reward": 14.674223899841309, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008599478751420975, + "policy/clipfrac_avg": 0.006935285869985819, + "policy/entropy_avg": 0.17142105102539062, + "step": 1075, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 73, + "val/ratio": 0.9890701770782471, + "val/ratio_var": 3.540362376952544e-05 + }, + { + "episode": 17216, + "epoch": 0.8694949494949495, + "eps": 0, + "loss/policy_avg": 0.11330074071884155, + "loss/value_avg": 0.47666221857070923, + "lr": 4.665812085953425e-07, + "objective/entropy": -62.084415435791016, + "objective/kl": 43.93402099609375, + "objective/non_score_reward": -2.1967010498046875, + "objective/rlhf_reward": 12.162673950195312, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009052088716998696, + "policy/clipfrac_avg": 0.0032377864699810743, + "policy/entropy_avg": 0.18408584594726562, + "step": 1076, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 588, + "val/ratio": 0.9876794815063477, + "val/ratio_var": 4.1342565964441746e-05 + }, + { + "episode": 17232, + "epoch": 0.8703030303030304, + "eps": 0, + "loss/policy_avg": -0.05885958671569824, + "loss/value_avg": 0.9560117125511169, + "lr": 4.6096321436504e-07, + "objective/entropy": -73.24485778808594, + "objective/kl": 39.06366729736328, + "objective/non_score_reward": -1.9531835317611694, + "objective/rlhf_reward": 13.5468168258667, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009686506818979979, + "policy/clipfrac_avg": 0.004011271521449089, + "policy/entropy_avg": 0.18913555145263672, + "step": 1077, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 200, + "val/ratio": 0.9870175719261169, + "val/ratio_var": 8.096115197986364e-05 + }, + { + "episode": 17248, + "epoch": 0.8711111111111111, + "eps": 0, + "loss/policy_avg": 0.18471674621105194, + "loss/value_avg": 0.37227392196655273, + "lr": 4.5537761293894535e-07, + "objective/entropy": -116.47470092773438, + "objective/kl": 37.137413024902344, + "objective/non_score_reward": -1.8568708896636963, + "objective/rlhf_reward": 14.861879348754883, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000727109843865037, + "policy/clipfrac_avg": 0.002056294586509466, + "policy/entropy_avg": 0.1612076759338379, + "step": 1078, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 178, + "val/ratio": 0.9902770519256592, + "val/ratio_var": 6.446443876484409e-05 + }, + { + "episode": 17264, + "epoch": 0.8719191919191919, + "eps": 0, + "loss/policy_avg": -0.16059936583042145, + "loss/value_avg": 0.507465124130249, + "lr": 4.4982444417866753e-07, + "objective/entropy": -93.82952880859375, + "objective/kl": 37.85641098022461, + "objective/non_score_reward": -1.8928205966949463, + "objective/rlhf_reward": 13.458742141723633, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0009164538932964206, + "policy/clipfrac_avg": 0.0050585465505719185, + "policy/entropy_avg": 0.15972977876663208, + "step": 1079, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 44, + "val/ratio": 0.989949643611908, + "val/ratio_var": 8.05418094387278e-05 + }, + { + "episode": 17280, + "epoch": 0.8727272727272727, + "eps": 0, + "loss/policy_avg": 0.024168267846107483, + "loss/value_avg": 0.5276412963867188, + "lr": 4.4430374771435245e-07, + "objective/entropy": -93.29814910888672, + "objective/kl": 36.883766174316406, + "objective/non_score_reward": -1.8441884517669678, + "objective/rlhf_reward": 13.952686309814453, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0010746137704700232, + "policy/clipfrac_avg": 0.005830212030559778, + "policy/entropy_avg": 0.1792011260986328, + "step": 1080, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 519, + "val/ratio": 0.989915132522583, + "val/ratio_var": 4.745582918985747e-05 + }, + { + "episode": 17296, + "epoch": 0.8735353535353535, + "eps": 0, + "loss/policy_avg": -0.09750235080718994, + "loss/value_avg": 0.4400123357772827, + "lr": 4.3881556294441516e-07, + "objective/entropy": -163.2279815673828, + "objective/kl": 42.45173645019531, + "objective/non_score_reward": -2.122586727142334, + "objective/rlhf_reward": 13.236787796020508, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007963902316987514, + "policy/clipfrac_avg": 0.0037931441329419613, + "policy/entropy_avg": 0.17099809646606445, + "step": 1081, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 294, + "val/ratio": 0.9886674880981445, + "val/ratio_var": 8.438028453383595e-05 + }, + { + "episode": 17312, + "epoch": 0.8743434343434343, + "eps": 0, + "loss/policy_avg": -0.0659959465265274, + "loss/value_avg": 0.2975969910621643, + "lr": 4.333599290352458e-07, + "objective/entropy": -63.06423568725586, + "objective/kl": 42.34510803222656, + "objective/non_score_reward": -2.117255210876465, + "objective/rlhf_reward": 14.288995742797852, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009070255910046399, + "policy/clipfrac_avg": 0.0022676163353025913, + "policy/entropy_avg": 0.18258285522460938, + "step": 1082, + "val/clipfrac_avg": 0.0035635964013636112, + "val/num_eos_tokens": 576, + "val/ratio": 0.9886539578437805, + "val/ratio_var": 6.365034641930833e-05 + }, + { + "episode": 17328, + "epoch": 0.8751515151515151, + "eps": 0, + "loss/policy_avg": -0.12663836777210236, + "loss/value_avg": 1.6143395900726318, + "lr": 4.279368849209381e-07, + "objective/entropy": -124.63575744628906, + "objective/kl": 34.96723937988281, + "objective/non_score_reward": -1.7483620643615723, + "objective/rlhf_reward": 15.900075912475586, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0008090031915344298, + "policy/clipfrac_avg": 0.0031590727157890797, + "policy/entropy_avg": 0.16525745391845703, + "step": 1083, + "val/clipfrac_avg": 0.02003205195069313, + "val/num_eos_tokens": 359, + "val/ratio": 0.9890646934509277, + "val/ratio_var": 7.246930908877403e-05 + }, + { + "episode": 17344, + "epoch": 0.8759595959595959, + "eps": 0, + "loss/policy_avg": 0.10138820856809616, + "loss/value_avg": 0.3533989191055298, + "lr": 4.2254646930300845e-07, + "objective/entropy": -55.43729019165039, + "objective/kl": 44.472023010253906, + "objective/non_score_reward": -2.2236011028289795, + "objective/rlhf_reward": 13.721712112426758, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009467223426327109, + "policy/clipfrac_avg": 0.0063713230192661285, + "policy/entropy_avg": 0.1949615478515625, + "step": 1084, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.986469030380249, + "val/ratio_var": 5.372995656216517e-05 + }, + { + "episode": 17360, + "epoch": 0.8767676767676768, + "eps": 0, + "loss/policy_avg": -0.16358020901679993, + "loss/value_avg": 0.7672687768936157, + "lr": 4.171887206501191e-07, + "objective/entropy": -80.793701171875, + "objective/kl": 35.82070541381836, + "objective/non_score_reward": -1.7910352945327759, + "objective/rlhf_reward": 13.552715301513672, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007973198080435395, + "policy/clipfrac_avg": 0.003301304765045643, + "policy/entropy_avg": 0.15000247955322266, + "step": 1085, + "val/clipfrac_avg": 0.0021186440717428923, + "val/num_eos_tokens": 403, + "val/ratio": 0.9896234273910522, + "val/ratio_var": 5.3207288146950305e-05 + }, + { + "episode": 17376, + "epoch": 0.8775757575757576, + "eps": 0, + "loss/policy_avg": 0.20687676966190338, + "loss/value_avg": 1.3663004636764526, + "lr": 4.1186367719780737e-07, + "objective/entropy": -46.82775115966797, + "objective/kl": 33.1654167175293, + "objective/non_score_reward": -1.6582709550857544, + "objective/rlhf_reward": 14.404230117797852, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008230544626712799, + "policy/clipfrac_avg": 0.0030164497438818216, + "policy/entropy_avg": 0.1555004119873047, + "step": 1086, + "val/clipfrac_avg": 0.0026466259732842445, + "val/num_eos_tokens": 47, + "val/ratio": 0.9892673492431641, + "val/ratio_var": 3.7006568163633347e-05 + }, + { + "episode": 17392, + "epoch": 0.8783838383838384, + "eps": 0, + "loss/policy_avg": 0.18984848260879517, + "loss/value_avg": 0.31801366806030273, + "lr": 4.0657137694820826e-07, + "objective/entropy": -66.93091583251953, + "objective/kl": 45.35851287841797, + "objective/non_score_reward": -2.26792573928833, + "objective/rlhf_reward": 14.122699737548828, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008988711051642895, + "policy/clipfrac_avg": 0.004365711007267237, + "policy/entropy_avg": 0.17530107498168945, + "step": 1087, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 363, + "val/ratio": 0.9879393577575684, + "val/ratio_var": 6.273078179219738e-05 + }, + { + "episode": 17408, + "epoch": 0.8791919191919192, + "eps": 0, + "loss/policy_avg": 0.14154401421546936, + "loss/value_avg": 0.296213299036026, + "lr": 4.013118576697844e-07, + "objective/entropy": -96.627197265625, + "objective/kl": 44.89594650268555, + "objective/non_score_reward": -2.244797468185425, + "objective/rlhf_reward": 14.028639793395996, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009503285982646048, + "policy/clipfrac_avg": 0.004713431932032108, + "policy/entropy_avg": 0.19094181060791016, + "step": 1088, + "val/clipfrac_avg": 0.004036702681332827, + "val/num_eos_tokens": 215, + "val/ratio": 0.9875110983848572, + "val/ratio_var": 3.645908509497531e-05 + }, + { + "episode": 17424, + "epoch": 0.88, + "eps": 0, + "loss/policy_avg": -0.2717769145965576, + "loss/value_avg": 0.6515541076660156, + "lr": 3.960851568970586e-07, + "objective/entropy": -81.71890258789062, + "objective/kl": 44.09088897705078, + "objective/non_score_reward": -2.2045445442199707, + "objective/rlhf_reward": 14.311080932617188, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008286006050184369, + "policy/clipfrac_avg": 0.0029842001385986805, + "policy/entropy_avg": 0.16189241409301758, + "step": 1089, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 391, + "val/ratio": 0.9892988204956055, + "val/ratio_var": 5.128037810209207e-05 + }, + { + "episode": 17440, + "epoch": 0.8808080808080808, + "eps": 0, + "loss/policy_avg": -0.09007427841424942, + "loss/value_avg": 0.174169659614563, + "lr": 3.908913119303448e-07, + "objective/entropy": -103.39527893066406, + "objective/kl": 45.33392333984375, + "objective/non_score_reward": -2.2666964530944824, + "objective/rlhf_reward": 15.030179023742676, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0008385264663957059, + "policy/clipfrac_avg": 0.0022599862422794104, + "policy/entropy_avg": 0.167741060256958, + "step": 1090, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 96, + "val/ratio": 0.9883161783218384, + "val/ratio_var": 4.988587534171529e-05 + }, + { + "episode": 17456, + "epoch": 0.8816161616161616, + "eps": 0, + "loss/policy_avg": 0.29356396198272705, + "loss/value_avg": 0.8743624687194824, + "lr": 3.8573035983548167e-07, + "objective/entropy": -82.11065673828125, + "objective/kl": 38.23064422607422, + "objective/non_score_reward": -1.9115324020385742, + "objective/rlhf_reward": 13.299405097961426, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008716207230463624, + "policy/clipfrac_avg": 0.0072744423523545265, + "policy/entropy_avg": 0.16114044189453125, + "step": 1091, + "val/clipfrac_avg": 0.0008928571478463709, + "val/num_eos_tokens": 91, + "val/ratio": 0.9882928133010864, + "val/ratio_var": 6.514011329272762e-05 + }, + { + "episode": 17472, + "epoch": 0.8824242424242424, + "eps": 0, + "loss/policy_avg": 0.02994067780673504, + "loss/value_avg": 0.792471170425415, + "lr": 3.8060233744356634e-07, + "objective/entropy": -12.753183364868164, + "objective/kl": 37.99050521850586, + "objective/non_score_reward": -1.8995252847671509, + "objective/rlhf_reward": 12.65125560760498, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0010071105789393187, + "policy/clipfrac_avg": 0.004005424678325653, + "policy/entropy_avg": 0.19425201416015625, + "step": 1092, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 14, + "val/ratio": 0.9855411052703857, + "val/ratio_var": 4.6690693125128746e-05 + }, + { + "episode": 17488, + "epoch": 0.8832323232323233, + "eps": 0, + "loss/policy_avg": 0.1304478645324707, + "loss/value_avg": 0.3748246431350708, + "lr": 3.7550728135069545e-07, + "objective/entropy": -36.16312789916992, + "objective/kl": 48.05560302734375, + "objective/non_score_reward": -2.402780532836914, + "objective/rlhf_reward": 14.300344467163086, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000979696400463581, + "policy/clipfrac_avg": 0.005794232711195946, + "policy/entropy_avg": 0.19660568237304688, + "step": 1093, + "val/clipfrac_avg": 0.0026041667442768812, + "val/num_eos_tokens": 66, + "val/ratio": 0.9865772128105164, + "val/ratio_var": 5.429132215795107e-05 + }, + { + "episode": 17504, + "epoch": 0.8840404040404041, + "eps": 0, + "loss/policy_avg": 0.07484120875597, + "loss/value_avg": 0.2602160573005676, + "lr": 3.704452279177012e-07, + "objective/entropy": -71.43814086914062, + "objective/kl": 45.28865051269531, + "objective/non_score_reward": -2.264432668685913, + "objective/rlhf_reward": 13.727754592895508, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000950466375797987, + "policy/clipfrac_avg": 0.0027152306865900755, + "policy/entropy_avg": 0.20662879943847656, + "step": 1094, + "val/clipfrac_avg": 0.0006720430101267993, + "val/num_eos_tokens": 494, + "val/ratio": 0.9866384863853455, + "val/ratio_var": 7.357261347351596e-05 + }, + { + "episode": 17520, + "epoch": 0.8848484848484849, + "eps": 0, + "loss/policy_avg": 0.10963790118694305, + "loss/value_avg": 0.7320235967636108, + "lr": 3.6541621326989183e-07, + "objective/entropy": -48.0823974609375, + "objective/kl": 39.92009353637695, + "objective/non_score_reward": -1.9960048198699951, + "objective/rlhf_reward": 12.980557441711426, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0010038036853075027, + "policy/clipfrac_avg": 0.004399286117404699, + "policy/entropy_avg": 0.19763946533203125, + "step": 1095, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 436, + "val/ratio": 0.987025260925293, + "val/ratio_var": 3.8341207982739434e-05 + }, + { + "episode": 17536, + "epoch": 0.8856565656565657, + "eps": 0, + "loss/policy_avg": 0.05299535021185875, + "loss/value_avg": 0.2717387080192566, + "lr": 3.6042027329679487e-07, + "objective/entropy": -51.004615783691406, + "objective/kl": 41.74889373779297, + "objective/non_score_reward": -2.087444543838501, + "objective/rlhf_reward": 14.545368194580078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010824615601450205, + "policy/clipfrac_avg": 0.00963993463665247, + "policy/entropy_avg": 0.20147705078125, + "step": 1096, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 345, + "val/ratio": 0.9875917434692383, + "val/ratio_var": 6.199166091391817e-05 + }, + { + "episode": 17552, + "epoch": 0.8864646464646465, + "eps": 0, + "loss/policy_avg": -0.08447069674730301, + "loss/value_avg": 0.38099169731140137, + "lr": 3.5545744365190204e-07, + "objective/entropy": -108.7376937866211, + "objective/kl": 41.82084655761719, + "objective/non_score_reward": -2.0910425186157227, + "objective/rlhf_reward": 14.158958435058594, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000758812646381557, + "policy/clipfrac_avg": 0.003666321514174342, + "policy/entropy_avg": 0.15442466735839844, + "step": 1097, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 244, + "val/ratio": 0.9902445077896118, + "val/ratio_var": 2.8433618354029022e-05 + }, + { + "episode": 17568, + "epoch": 0.8872727272727273, + "eps": 0, + "loss/policy_avg": -0.47139230370521545, + "loss/value_avg": 1.4234567880630493, + "lr": 3.5052775975241203e-07, + "objective/entropy": -131.56527709960938, + "objective/kl": 33.65018081665039, + "objective/non_score_reward": -1.682509183883667, + "objective/rlhf_reward": 13.74717903137207, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009085559286177158, + "policy/clipfrac_avg": 0.004985325504094362, + "policy/entropy_avg": 0.16993939876556396, + "step": 1098, + "val/clipfrac_avg": 0.00024242099607363343, + "val/num_eos_tokens": 1220, + "val/ratio": 0.9888011813163757, + "val/ratio_var": 7.847072265576571e-05 + }, + { + "episode": 17584, + "epoch": 0.888080808080808, + "eps": 0, + "loss/policy_avg": 0.16145119071006775, + "loss/value_avg": 0.5360366106033325, + "lr": 3.4563125677897936e-07, + "objective/entropy": -42.34114074707031, + "objective/kl": 37.358299255371094, + "objective/non_score_reward": -1.867915153503418, + "objective/rlhf_reward": 14.428959846496582, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.001106644980609417, + "policy/clipfrac_avg": 0.0073922015726566315, + "policy/entropy_avg": 0.216278076171875, + "step": 1099, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 172, + "val/ratio": 0.9858206510543823, + "val/ratio_var": 6.1017286498099566e-05 + }, + { + "episode": 17600, + "epoch": 0.8888888888888888, + "eps": 0, + "loss/policy_avg": 0.043675780296325684, + "loss/value_avg": 0.5687451362609863, + "lr": 3.4076796967546176e-07, + "objective/entropy": -32.0341682434082, + "objective/kl": 43.24127197265625, + "objective/non_score_reward": -2.1620638370513916, + "objective/rlhf_reward": 12.939498901367188, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010629122843965888, + "policy/clipfrac_avg": 0.003724365960806608, + "policy/entropy_avg": 0.20337867736816406, + "step": 1100, + "val/clipfrac_avg": 0.005150945391505957, + "val/num_eos_tokens": 106, + "val/ratio": 0.9857684969902039, + "val/ratio_var": 4.0027771319728345e-05 + }, + { + "episode": 17616, + "epoch": 0.8896969696969697, + "eps": 0, + "loss/policy_avg": -0.0018694326281547546, + "loss/value_avg": 0.43309682607650757, + "lr": 3.359379331486762e-07, + "objective/entropy": -65.23380279541016, + "objective/kl": 40.037052154541016, + "objective/non_score_reward": -2.001852512359619, + "objective/rlhf_reward": 13.349709510803223, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010041861096397042, + "policy/clipfrac_avg": 0.004063514992594719, + "policy/entropy_avg": 0.19306087493896484, + "step": 1101, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 14, + "val/ratio": 0.9867812395095825, + "val/ratio_var": 5.167184644960798e-05 + }, + { + "episode": 17632, + "epoch": 0.8905050505050505, + "eps": 0, + "loss/policy_avg": 0.048976313322782516, + "loss/value_avg": 0.3672630190849304, + "lr": 3.311411816681426e-07, + "objective/entropy": -81.65939331054688, + "objective/kl": 41.81010055541992, + "objective/non_score_reward": -2.0905051231384277, + "objective/rlhf_reward": 14.393869400024414, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000925385276786983, + "policy/clipfrac_avg": 0.004391741007566452, + "policy/entropy_avg": 0.1823406219482422, + "step": 1102, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 128, + "val/ratio": 0.9886959791183472, + "val/ratio_var": 5.684143616235815e-05 + }, + { + "episode": 17648, + "epoch": 0.8913131313131313, + "eps": 0, + "loss/policy_avg": 0.02880220115184784, + "loss/value_avg": 0.3456343114376068, + "lr": 3.263777494658449e-07, + "objective/entropy": -97.10613250732422, + "objective/kl": 39.38318634033203, + "objective/non_score_reward": -1.9691593647003174, + "objective/rlhf_reward": 14.226152420043945, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008451448520645499, + "policy/clipfrac_avg": 0.0035835830494761467, + "policy/entropy_avg": 0.18262100219726562, + "step": 1103, + "val/clipfrac_avg": 0.0004976114723831415, + "val/num_eos_tokens": 81, + "val/ratio": 0.9880388975143433, + "val/ratio_var": 4.812321640201844e-05 + }, + { + "episode": 17664, + "epoch": 0.8921212121212121, + "eps": 0, + "loss/policy_avg": -0.15117420256137848, + "loss/value_avg": 1.1375534534454346, + "lr": 3.216476705359839e-07, + "objective/entropy": -138.00392150878906, + "objective/kl": 39.81349182128906, + "objective/non_score_reward": -1.990674614906311, + "objective/rlhf_reward": 13.14213752746582, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009344244026578963, + "policy/clipfrac_avg": 0.004199904855340719, + "policy/entropy_avg": 0.1756887435913086, + "step": 1104, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 173, + "val/ratio": 0.9869411587715149, + "val/ratio_var": 8.181409793905914e-05 + }, + { + "episode": 17680, + "epoch": 0.8929292929292929, + "eps": 0, + "loss/policy_avg": -0.40621763467788696, + "loss/value_avg": 0.46248072385787964, + "lr": 3.1695097863473355e-07, + "objective/entropy": -191.18865966796875, + "objective/kl": 38.82806396484375, + "objective/non_score_reward": -1.9414031505584717, + "objective/rlhf_reward": 13.941408157348633, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0005901164840906858, + "policy/clipfrac_avg": 0.005352393724024296, + "policy/entropy_avg": 0.1206204891204834, + "step": 1105, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 236, + "val/ratio": 0.9924303293228149, + "val/ratio_var": 4.216426896164194e-05 + }, + { + "episode": 17696, + "epoch": 0.8937373737373737, + "eps": 0, + "loss/policy_avg": -0.11594279855489731, + "loss/value_avg": 0.7615197896957397, + "lr": 3.122877072800046e-07, + "objective/entropy": -87.4638900756836, + "objective/kl": 37.186309814453125, + "objective/non_score_reward": -1.8593155145645142, + "objective/rlhf_reward": 15.757871627807617, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0008562589064240456, + "policy/clipfrac_avg": 0.0030250055715441704, + "policy/entropy_avg": 0.1770009994506836, + "step": 1106, + "val/clipfrac_avg": 8.561644062865525e-05, + "val/num_eos_tokens": 557, + "val/ratio": 0.9886528253555298, + "val/ratio_var": 7.094601460266858e-05 + }, + { + "episode": 17712, + "epoch": 0.8945454545454545, + "eps": 0, + "loss/policy_avg": -0.12702879309654236, + "loss/value_avg": 0.3219323754310608, + "lr": 3.076578897511978e-07, + "objective/entropy": -110.63394927978516, + "objective/kl": 39.78605651855469, + "objective/non_score_reward": -1.9893031120300293, + "objective/rlhf_reward": 14.362258911132812, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007805367931723595, + "policy/clipfrac_avg": 0.0034728883765637875, + "policy/entropy_avg": 0.14988577365875244, + "step": 1107, + "val/clipfrac_avg": 0.00021701389050576836, + "val/num_eos_tokens": 344, + "val/ratio": 0.9894638657569885, + "val/ratio_var": 6.243310053832829e-05 + }, + { + "episode": 17728, + "epoch": 0.8953535353535353, + "eps": 0, + "loss/policy_avg": -0.030154824256896973, + "loss/value_avg": 0.6115067005157471, + "lr": 3.0306155908897626e-07, + "objective/entropy": -74.73516845703125, + "objective/kl": 35.881317138671875, + "objective/non_score_reward": -1.7940657138824463, + "objective/rlhf_reward": 14.002809524536133, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009087615762837231, + "policy/clipfrac_avg": 0.004955953918397427, + "policy/entropy_avg": 0.1930408477783203, + "step": 1108, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 300, + "val/ratio": 0.9869700074195862, + "val/ratio_var": 6.0412487073335797e-05 + }, + { + "episode": 17744, + "epoch": 0.8961616161616162, + "eps": 0, + "loss/policy_avg": -0.01582721620798111, + "loss/value_avg": 0.5954544544219971, + "lr": 2.984987480950191e-07, + "objective/entropy": -105.74530029296875, + "objective/kl": 33.56185531616211, + "objective/non_score_reward": -1.6780929565429688, + "objective/rlhf_reward": 12.939094543457031, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009151956764981151, + "policy/clipfrac_avg": 0.004283002577722073, + "policy/entropy_avg": 0.18004417419433594, + "step": 1109, + "val/clipfrac_avg": 0.001953125, + "val/num_eos_tokens": 107, + "val/ratio": 0.9886264801025391, + "val/ratio_var": 6.606351962545887e-05 + }, + { + "episode": 17760, + "epoch": 0.896969696969697, + "eps": 0, + "loss/policy_avg": -0.0613558366894722, + "loss/value_avg": 0.37346547842025757, + "lr": 2.939694893317979e-07, + "objective/entropy": -105.83699798583984, + "objective/kl": 40.10255432128906, + "objective/non_score_reward": -2.0051279067993164, + "objective/rlhf_reward": 13.768309593200684, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009933009278029203, + "policy/clipfrac_avg": 0.002607613569125533, + "policy/entropy_avg": 0.19908523559570312, + "step": 1110, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 189, + "val/ratio": 0.9867061376571655, + "val/ratio_var": 8.13972728792578e-05 + }, + { + "episode": 17776, + "epoch": 0.8977777777777778, + "eps": 0, + "loss/policy_avg": 0.0873388946056366, + "loss/value_avg": 0.6430274844169617, + "lr": 2.894738151223331e-07, + "objective/entropy": -69.66899108886719, + "objective/kl": 45.03960418701172, + "objective/non_score_reward": -2.2519803047180176, + "objective/rlhf_reward": 15.544894218444824, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0010472675785422325, + "policy/clipfrac_avg": 0.0032264026813209057, + "policy/entropy_avg": 0.20671892166137695, + "step": 1111, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 714, + "val/ratio": 0.9859870672225952, + "val/ratio_var": 6.778486567782238e-05 + }, + { + "episode": 17792, + "epoch": 0.8985858585858586, + "eps": 0, + "loss/policy_avg": -0.13560356199741364, + "loss/value_avg": 0.6080023646354675, + "lr": 2.850117575499739e-07, + "objective/entropy": -89.71099090576172, + "objective/kl": 39.79743957519531, + "objective/non_score_reward": -1.9898720979690552, + "objective/rlhf_reward": 16.385128021240234, + "objective/scores": 18.375, + "policy/approxkl_avg": 0.00102079042699188, + "policy/clipfrac_avg": 0.004375361371785402, + "policy/entropy_avg": 0.19225120544433594, + "step": 1112, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 296, + "val/ratio": 0.9872239828109741, + "val/ratio_var": 7.309510692721233e-05 + }, + { + "episode": 17808, + "epoch": 0.8993939393939394, + "eps": 0, + "loss/policy_avg": -0.33778393268585205, + "loss/value_avg": 0.4939768612384796, + "lr": 2.8058334845816214e-07, + "objective/entropy": -31.377525329589844, + "objective/kl": 48.40119934082031, + "objective/non_score_reward": -2.4200599193573, + "objective/rlhf_reward": 15.626814842224121, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010773141402751207, + "policy/clipfrac_avg": 0.007447387091815472, + "policy/entropy_avg": 0.2159743309020996, + "step": 1113, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 28, + "val/ratio": 0.9851824641227722, + "val/ratio_var": 7.278792327269912e-05 + }, + { + "episode": 17824, + "epoch": 0.9002020202020202, + "eps": 0, + "loss/policy_avg": -0.1028350442647934, + "loss/value_avg": 0.5992170572280884, + "lr": 2.761886194502089e-07, + "objective/entropy": -30.16031265258789, + "objective/kl": 37.62234115600586, + "objective/non_score_reward": -1.8811171054840088, + "objective/rlhf_reward": 12.79857063293457, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010548201389610767, + "policy/clipfrac_avg": 0.0043435124680399895, + "policy/entropy_avg": 0.19416046142578125, + "step": 1114, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 115, + "val/ratio": 0.9875701665878296, + "val/ratio_var": 4.415150760905817e-05 + }, + { + "episode": 17840, + "epoch": 0.901010101010101, + "eps": 0, + "loss/policy_avg": 0.15831202268600464, + "loss/value_avg": 0.5015832185745239, + "lr": 2.71827601889067e-07, + "objective/entropy": -98.54946899414062, + "objective/kl": 44.19203567504883, + "objective/non_score_reward": -2.209601879119873, + "objective/rlhf_reward": 12.516960144042969, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0008550393395125866, + "policy/clipfrac_avg": 0.0035258647985756397, + "policy/entropy_avg": 0.17531871795654297, + "step": 1115, + "val/clipfrac_avg": 9.245562250725925e-05, + "val/num_eos_tokens": 277, + "val/ratio": 0.9883965253829956, + "val/ratio_var": 6.70754598104395e-05 + }, + { + "episode": 17856, + "epoch": 0.9018181818181819, + "eps": 0, + "loss/policy_avg": -0.0883672684431076, + "loss/value_avg": 0.38530808687210083, + "lr": 2.6750032689710604e-07, + "objective/entropy": -89.13906860351562, + "objective/kl": 40.806068420410156, + "objective/non_score_reward": -2.0403032302856445, + "objective/rlhf_reward": 14.842509269714355, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009749368182383478, + "policy/clipfrac_avg": 0.004587238188832998, + "policy/entropy_avg": 0.1862497329711914, + "step": 1116, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 42, + "val/ratio": 0.9878228902816772, + "val/ratio_var": 6.971527182031423e-05 + }, + { + "episode": 17872, + "epoch": 0.9026262626262627, + "eps": 0, + "loss/policy_avg": 0.35861504077911377, + "loss/value_avg": 0.47818782925605774, + "lr": 2.6320682535589557e-07, + "objective/entropy": -94.74272155761719, + "objective/kl": 39.73638916015625, + "objective/non_score_reward": -1.9868193864822388, + "objective/rlhf_reward": 14.48974323272705, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.001014375826343894, + "policy/clipfrac_avg": 0.009114467538893223, + "policy/entropy_avg": 0.18755674362182617, + "step": 1117, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 316, + "val/ratio": 0.9870187044143677, + "val/ratio_var": 6.580681656487286e-05 + }, + { + "episode": 17888, + "epoch": 0.9034343434343435, + "eps": 0, + "loss/policy_avg": 0.03886333107948303, + "loss/value_avg": 0.5049983263015747, + "lr": 2.589471279059791e-07, + "objective/entropy": -60.80832290649414, + "objective/kl": 38.03620147705078, + "objective/non_score_reward": -1.9018102884292603, + "objective/rlhf_reward": 14.418501853942871, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009664712124504149, + "policy/clipfrac_avg": 0.0054984488524496555, + "policy/entropy_avg": 0.18600749969482422, + "step": 1118, + "val/clipfrac_avg": 0.00028935185400769114, + "val/num_eos_tokens": 538, + "val/ratio": 0.9878861904144287, + "val/ratio_var": 8.505670120939612e-05 + }, + { + "episode": 17904, + "epoch": 0.9042424242424243, + "eps": 0, + "loss/policy_avg": -0.30915409326553345, + "loss/value_avg": 0.6310875415802002, + "lr": 2.547212649466568e-07, + "objective/entropy": -66.79815673828125, + "objective/kl": 40.48139190673828, + "objective/non_score_reward": -2.0240697860717773, + "objective/rlhf_reward": 13.483742713928223, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0010647093877196312, + "policy/clipfrac_avg": 0.005042470060288906, + "policy/entropy_avg": 0.2105545997619629, + "step": 1119, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 341, + "val/ratio": 0.9860644340515137, + "val/ratio_var": 7.680879207327962e-05 + }, + { + "episode": 17920, + "epoch": 0.9050505050505051, + "eps": 0, + "loss/policy_avg": -0.0354868620634079, + "loss/value_avg": 0.43115413188934326, + "lr": 2.5052926663577006e-07, + "objective/entropy": -109.19269561767578, + "objective/kl": 37.41370391845703, + "objective/non_score_reward": -1.8706854581832886, + "objective/rlhf_reward": 15.559001922607422, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0008096316596493125, + "policy/clipfrac_avg": 0.007008845917880535, + "policy/entropy_avg": 0.1526775360107422, + "step": 1120, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 27, + "val/ratio": 0.990079402923584, + "val/ratio_var": 6.870418292237446e-05 + }, + { + "episode": 17936, + "epoch": 0.9058585858585858, + "eps": 0, + "loss/policy_avg": -0.10384486615657806, + "loss/value_avg": 0.5555015802383423, + "lr": 2.4637116288948606e-07, + "objective/entropy": -81.66447448730469, + "objective/kl": 34.01490020751953, + "objective/non_score_reward": -1.7007449865341187, + "objective/rlhf_reward": 14.06488037109375, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011045493884012103, + "policy/clipfrac_avg": 0.004975738935172558, + "policy/entropy_avg": 0.19969654083251953, + "step": 1121, + "val/clipfrac_avg": 0.00020559210679493845, + "val/num_eos_tokens": 424, + "val/ratio": 0.9861068725585938, + "val/ratio_var": 0.0001140525855589658 + }, + { + "episode": 17952, + "epoch": 0.9066666666666666, + "eps": 0, + "loss/policy_avg": 0.08499134331941605, + "loss/value_avg": 0.4820760488510132, + "lr": 2.4224698338208344e-07, + "objective/entropy": -51.88709259033203, + "objective/kl": 36.03460693359375, + "objective/non_score_reward": -1.8017305135726929, + "objective/rlhf_reward": 13.284207344055176, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007865545339882374, + "policy/clipfrac_avg": 0.005190032534301281, + "policy/entropy_avg": 0.1644287109375, + "step": 1122, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 97, + "val/ratio": 0.9896594882011414, + "val/ratio_var": 3.428427953622304e-05 + }, + { + "episode": 17968, + "epoch": 0.9074747474747474, + "eps": 0, + "loss/policy_avg": -0.17406421899795532, + "loss/value_avg": 0.5896267294883728, + "lr": 2.3815675754573885e-07, + "objective/entropy": -86.18510437011719, + "objective/kl": 43.59098815917969, + "objective/non_score_reward": -2.179549217224121, + "objective/rlhf_reward": 14.422013282775879, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000971743487752974, + "policy/clipfrac_avg": 0.006423149257898331, + "policy/entropy_avg": 0.18558311462402344, + "step": 1123, + "val/clipfrac_avg": 0.0003633720916695893, + "val/num_eos_tokens": 28, + "val/ratio": 0.9873388409614563, + "val/ratio_var": 6.847102486062795e-05 + }, + { + "episode": 17984, + "epoch": 0.9082828282828282, + "eps": 0, + "loss/policy_avg": 0.003323908895254135, + "loss/value_avg": 0.4784785807132721, + "lr": 2.3410051457032179e-07, + "objective/entropy": -15.714594841003418, + "objective/kl": 46.467933654785156, + "objective/non_score_reward": -2.323396682739258, + "objective/rlhf_reward": 13.957853317260742, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009007846238091588, + "policy/clipfrac_avg": 0.004247956909239292, + "policy/entropy_avg": 0.18125152587890625, + "step": 1124, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 91, + "val/ratio": 0.9878742694854736, + "val/ratio_var": 4.734889080282301e-05 + }, + { + "episode": 18000, + "epoch": 0.9090909090909091, + "eps": 0, + "loss/policy_avg": -0.34681111574172974, + "loss/value_avg": 1.8761348724365234, + "lr": 2.3007828340318117e-07, + "objective/entropy": -78.62959289550781, + "objective/kl": 28.815372467041016, + "objective/non_score_reward": -1.4407685995101929, + "objective/rlhf_reward": 13.76235580444336, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0007939373608678579, + "policy/clipfrac_avg": 0.0023933430202305317, + "policy/entropy_avg": 0.14340460300445557, + "step": 1125, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 162, + "val/ratio": 0.9906671047210693, + "val/ratio_var": 8.598795102443546e-05 + }, + { + "episode": 18016, + "epoch": 0.9098989898989899, + "eps": 0, + "loss/policy_avg": -0.10182956606149673, + "loss/value_avg": 0.9262669682502747, + "lr": 2.2609009274894124e-07, + "objective/entropy": -84.69230651855469, + "objective/kl": 41.48188018798828, + "objective/non_score_reward": -2.07409405708313, + "objective/rlhf_reward": 14.402467727661133, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0007599142845720053, + "policy/clipfrac_avg": 0.004918557591736317, + "policy/entropy_avg": 0.14319229125976562, + "step": 1126, + "val/clipfrac_avg": 0.0003810975467786193, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904208779335022, + "val/ratio_var": 5.685169162461534e-05 + }, + { + "episode": 18032, + "epoch": 0.9107070707070707, + "eps": 0, + "loss/policy_avg": -0.05897387117147446, + "loss/value_avg": 0.6550663709640503, + "lr": 2.2213597106929608e-07, + "objective/entropy": 6.487635612487793, + "objective/kl": 36.32238006591797, + "objective/non_score_reward": -1.8161190748214722, + "objective/rlhf_reward": 12.402630805969238, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0012117292499169707, + "policy/clipfrac_avg": 0.003925529308617115, + "policy/entropy_avg": 0.22541046142578125, + "step": 1127, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 48, + "val/ratio": 0.9840712547302246, + "val/ratio_var": 4.843049100600183e-05 + }, + { + "episode": 18048, + "epoch": 0.9115151515151515, + "eps": 0, + "loss/policy_avg": -0.2118905931711197, + "loss/value_avg": 0.2701466381549835, + "lr": 2.1821594658280932e-07, + "objective/entropy": -81.51445007324219, + "objective/kl": 43.229087829589844, + "objective/non_score_reward": -2.161454439163208, + "objective/rlhf_reward": 14.604171752929688, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.000821998284664005, + "policy/clipfrac_avg": 0.003186737420037389, + "policy/entropy_avg": 0.16611194610595703, + "step": 1128, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 379, + "val/ratio": 0.9885213971138, + "val/ratio_var": 5.360436989576556e-05 + }, + { + "episode": 18064, + "epoch": 0.9123232323232323, + "eps": 0, + "loss/policy_avg": 0.07222969084978104, + "loss/value_avg": 0.13441604375839233, + "lr": 2.1433004726470695e-07, + "objective/entropy": -90.69831085205078, + "objective/kl": 43.020301818847656, + "objective/non_score_reward": -2.151015281677246, + "objective/rlhf_reward": 15.84117317199707, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0008969155605882406, + "policy/clipfrac_avg": 0.004066394176334143, + "policy/entropy_avg": 0.18462800979614258, + "step": 1129, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 122, + "val/ratio": 0.9881325364112854, + "val/ratio_var": 5.7613899116404355e-05 + }, + { + "episode": 18080, + "epoch": 0.9131313131313131, + "eps": 0, + "loss/policy_avg": 0.030582871288061142, + "loss/value_avg": 0.32031866908073425, + "lr": 2.1047830084668264e-07, + "objective/entropy": -97.6414566040039, + "objective/kl": 39.517765045166016, + "objective/non_score_reward": -1.9758882522583008, + "objective/rlhf_reward": 13.8444242477417, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0008600642322562635, + "policy/clipfrac_avg": 0.002656108234077692, + "policy/entropy_avg": 0.16113853454589844, + "step": 1130, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 730, + "val/ratio": 0.9883853793144226, + "val/ratio_var": 6.297288200585172e-05 + }, + { + "episode": 18096, + "epoch": 0.9139393939393939, + "eps": 0, + "loss/policy_avg": 0.004439752548933029, + "loss/value_avg": 0.5337626934051514, + "lr": 2.0666073481669714e-07, + "objective/entropy": -67.81413269042969, + "objective/kl": 34.70388412475586, + "objective/non_score_reward": -1.735194206237793, + "objective/rlhf_reward": 14.077305793762207, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009027072228491306, + "policy/clipfrac_avg": 0.004612314514815807, + "policy/entropy_avg": 0.18857192993164062, + "step": 1131, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 89, + "val/ratio": 0.9875516891479492, + "val/ratio_var": 7.778128201607615e-05 + }, + { + "episode": 18112, + "epoch": 0.9147474747474748, + "eps": 0, + "loss/policy_avg": -0.0649208128452301, + "loss/value_avg": 0.2573353052139282, + "lr": 2.0287737641878358e-07, + "objective/entropy": -57.457305908203125, + "objective/kl": 41.89903259277344, + "objective/non_score_reward": -2.094951629638672, + "objective/rlhf_reward": 15.764423370361328, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.001014683861285448, + "policy/clipfrac_avg": 0.005427089519798756, + "policy/entropy_avg": 0.2117767333984375, + "step": 1132, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 81, + "val/ratio": 0.985901415348053, + "val/ratio_var": 5.110340498504229e-05 + }, + { + "episode": 18128, + "epoch": 0.9155555555555556, + "eps": 0, + "loss/policy_avg": -0.03162732720375061, + "loss/value_avg": 0.41586339473724365, + "lr": 1.991282526528543e-07, + "objective/entropy": -32.120155334472656, + "objective/kl": 47.3048095703125, + "objective/non_score_reward": -2.3652405738830566, + "objective/rlhf_reward": 13.720697402954102, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010292984079569578, + "policy/clipfrac_avg": 0.004157859366387129, + "policy/entropy_avg": 0.20333099365234375, + "step": 1133, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 51, + "val/ratio": 0.9864168167114258, + "val/ratio_var": 5.427391442935914e-05 + }, + { + "episode": 18144, + "epoch": 0.9163636363636364, + "eps": 0, + "loss/policy_avg": 0.0864434763789177, + "loss/value_avg": 0.2746288478374481, + "lr": 1.9541339027450256e-07, + "objective/entropy": -68.75173950195312, + "objective/kl": 41.02958679199219, + "objective/non_score_reward": -2.0514793395996094, + "objective/rlhf_reward": 13.80789566040039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009240241488441825, + "policy/clipfrac_avg": 0.0033770925365388393, + "policy/entropy_avg": 0.182037353515625, + "step": 1134, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 236, + "val/ratio": 0.9879450798034668, + "val/ratio_var": 5.012234760215506e-05 + }, + { + "episode": 18160, + "epoch": 0.9171717171717172, + "eps": 0, + "loss/policy_avg": 0.06688633561134338, + "loss/value_avg": 0.4891657829284668, + "lr": 1.9173281579481896e-07, + "objective/entropy": -23.645580291748047, + "objective/kl": 38.17439270019531, + "objective/non_score_reward": -1.9087196588516235, + "objective/rlhf_reward": 15.474092483520508, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.000994181027635932, + "policy/clipfrac_avg": 0.0035517350770533085, + "policy/entropy_avg": 0.21059799194335938, + "step": 1135, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4, + "val/ratio": 0.9864646196365356, + "val/ratio_var": 6.11128198215738e-05 + }, + { + "episode": 18176, + "epoch": 0.917979797979798, + "eps": 0, + "loss/policy_avg": 0.09911377727985382, + "loss/value_avg": 0.18598923087120056, + "lr": 1.8808655548019638e-07, + "objective/entropy": -43.90042495727539, + "objective/kl": 43.086509704589844, + "objective/non_score_reward": -2.154325485229492, + "objective/rlhf_reward": 14.970674514770508, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0010021792259067297, + "policy/clipfrac_avg": 0.00640224339440465, + "policy/entropy_avg": 0.19201087951660156, + "step": 1136, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 455, + "val/ratio": 0.9868404865264893, + "val/ratio_var": 3.5931021557189524e-05 + }, + { + "episode": 18192, + "epoch": 0.9187878787878788, + "eps": 0, + "loss/policy_avg": -0.0433524064719677, + "loss/value_avg": 0.6105636358261108, + "lr": 1.8447463535214872e-07, + "objective/entropy": -149.0535125732422, + "objective/kl": 41.71107482910156, + "objective/non_score_reward": -2.0855536460876465, + "objective/rlhf_reward": 15.117570877075195, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0006844973540864885, + "policy/clipfrac_avg": 0.004837099928408861, + "policy/entropy_avg": 0.1427140235900879, + "step": 1137, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 356, + "val/ratio": 0.990210235118866, + "val/ratio_var": 5.7032579206861556e-05 + }, + { + "episode": 18208, + "epoch": 0.9195959595959596, + "eps": 0, + "loss/policy_avg": -0.038881611078977585, + "loss/value_avg": 0.2666724920272827, + "lr": 1.8089708118711525e-07, + "objective/entropy": -55.38002395629883, + "objective/kl": 38.607025146484375, + "objective/non_score_reward": -1.9303512573242188, + "objective/rlhf_reward": 14.397773742675781, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009189241100102663, + "policy/clipfrac_avg": 0.003879917087033391, + "policy/entropy_avg": 0.15883827209472656, + "step": 1138, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 41, + "val/ratio": 0.9893890619277954, + "val/ratio_var": 5.718938700738363e-05 + }, + { + "episode": 18224, + "epoch": 0.9204040404040404, + "eps": 0, + "loss/policy_avg": 0.06429219990968704, + "loss/value_avg": 0.8066933155059814, + "lr": 1.7735391851628814e-07, + "objective/entropy": -44.06977462768555, + "objective/kl": 34.568389892578125, + "objective/non_score_reward": -1.728419542312622, + "objective/rlhf_reward": 14.62314224243164, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008048653253354132, + "policy/clipfrac_avg": 0.006366053596138954, + "policy/entropy_avg": 0.18921661376953125, + "step": 1139, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 33, + "val/ratio": 0.9889757037162781, + "val/ratio_var": 3.707074938574806e-05 + }, + { + "episode": 18240, + "epoch": 0.9212121212121213, + "eps": 0, + "loss/policy_avg": 0.14161835610866547, + "loss/value_avg": 0.728713870048523, + "lr": 1.7384517262542255e-07, + "objective/entropy": -85.9615707397461, + "objective/kl": 39.761756896972656, + "objective/non_score_reward": -1.988087773323059, + "objective/rlhf_reward": 14.47284984588623, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009051311062648892, + "policy/clipfrac_avg": 0.0030989747028797865, + "policy/entropy_avg": 0.15839195251464844, + "step": 1140, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 59, + "val/ratio": 0.9888681173324585, + "val/ratio_var": 7.059999916236848e-05 + }, + { + "episode": 18256, + "epoch": 0.9220202020202021, + "eps": 0, + "loss/policy_avg": 0.10231739282608032, + "loss/value_avg": 0.7710623145103455, + "lr": 1.7037086855465902e-07, + "objective/entropy": -50.15681457519531, + "objective/kl": 42.769935607910156, + "objective/non_score_reward": -2.1384968757629395, + "objective/rlhf_reward": 14.103690147399902, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009481002343818545, + "policy/clipfrac_avg": 0.0051858918741345406, + "policy/entropy_avg": 0.2104654312133789, + "step": 1141, + "val/clipfrac_avg": 2.8775322789442725e-05, + "val/num_eos_tokens": 158, + "val/ratio": 0.9877794981002808, + "val/ratio_var": 6.828846380813047e-05 + }, + { + "episode": 18272, + "epoch": 0.9228282828282828, + "eps": 0, + "loss/policy_avg": 0.06746610999107361, + "loss/value_avg": 0.32568371295928955, + "lr": 1.6693103109834408e-07, + "objective/entropy": -83.49034118652344, + "objective/kl": 37.26749801635742, + "objective/non_score_reward": -1.863374948501587, + "objective/rlhf_reward": 13.496000289916992, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008203347679227591, + "policy/clipfrac_avg": 0.007063254248350859, + "policy/entropy_avg": 0.16310882568359375, + "step": 1142, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 109, + "val/ratio": 0.9898014068603516, + "val/ratio_var": 4.5457767555490136e-05 + }, + { + "episode": 18288, + "epoch": 0.9236363636363636, + "eps": 0, + "loss/policy_avg": -0.1702660322189331, + "loss/value_avg": 0.35972562432289124, + "lr": 1.6352568480485277e-07, + "objective/entropy": -22.768522262573242, + "objective/kl": 45.40270233154297, + "objective/non_score_reward": -2.2701351642608643, + "objective/rlhf_reward": 14.222051620483398, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.001096026157028973, + "policy/clipfrac_avg": 0.004881397821009159, + "policy/entropy_avg": 0.21602249145507812, + "step": 1143, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 105, + "val/ratio": 0.9855197668075562, + "val/ratio_var": 3.710871169460006e-05 + }, + { + "episode": 18304, + "epoch": 0.9244444444444444, + "eps": 0, + "loss/policy_avg": 0.19920386373996735, + "loss/value_avg": 0.5300213098526001, + "lr": 1.6015485397641696e-07, + "objective/entropy": -49.97838592529297, + "objective/kl": 33.057132720947266, + "objective/non_score_reward": -1.6528568267822266, + "objective/rlhf_reward": 15.800268173217773, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0009639778290875256, + "policy/clipfrac_avg": 0.00445273332297802, + "policy/entropy_avg": 0.180816650390625, + "step": 1144, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 508, + "val/ratio": 0.9873702526092529, + "val/ratio_var": 4.910973075311631e-05 + }, + { + "episode": 18320, + "epoch": 0.9252525252525252, + "eps": 0, + "loss/policy_avg": -0.09338352829217911, + "loss/value_avg": 0.23753158748149872, + "lr": 1.5681856266894612e-07, + "objective/entropy": -32.52317810058594, + "objective/kl": 46.23842239379883, + "objective/non_score_reward": -2.3119211196899414, + "objective/rlhf_reward": 14.336517333984375, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009981514886021614, + "policy/clipfrac_avg": 0.002960982732474804, + "policy/entropy_avg": 0.20490646362304688, + "step": 1145, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 284, + "val/ratio": 0.9861640334129333, + "val/ratio_var": 5.2834289817838e-05 + }, + { + "episode": 18336, + "epoch": 0.926060606060606, + "eps": 0, + "loss/policy_avg": -0.194179505109787, + "loss/value_avg": 0.8305935859680176, + "lr": 1.5351683469185973e-07, + "objective/entropy": -102.07046508789062, + "objective/kl": 31.8308162689209, + "objective/non_score_reward": -1.5915409326553345, + "objective/rlhf_reward": 14.752208709716797, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008943321881815791, + "policy/clipfrac_avg": 0.006945772562175989, + "policy/entropy_avg": 0.16139507293701172, + "step": 1146, + "val/clipfrac_avg": 0.00017361111531499773, + "val/num_eos_tokens": 780, + "val/ratio": 0.9883906245231628, + "val/ratio_var": 7.156646461226046e-05 + }, + { + "episode": 18352, + "epoch": 0.9268686868686868, + "eps": 0, + "loss/policy_avg": -0.14431902766227722, + "loss/value_avg": 1.0912277698516846, + "lr": 1.5024969360791564e-07, + "objective/entropy": -73.28153991699219, + "objective/kl": 31.00863265991211, + "objective/non_score_reward": -1.550431728363037, + "objective/rlhf_reward": 13.941755294799805, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000826484989374876, + "policy/clipfrac_avg": 0.004950974136590958, + "policy/entropy_avg": 0.15498733520507812, + "step": 1147, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 441, + "val/ratio": 0.9889597296714783, + "val/ratio_var": 6.787945312680677e-05 + }, + { + "episode": 18368, + "epoch": 0.9276767676767677, + "eps": 0, + "loss/policy_avg": -0.03664656728506088, + "loss/value_avg": 0.5489372611045837, + "lr": 1.4701716273304524e-07, + "objective/entropy": -56.737247467041016, + "objective/kl": 37.2110710144043, + "objective/non_score_reward": -1.860553503036499, + "objective/rlhf_reward": 13.061321258544922, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009301225654780865, + "policy/clipfrac_avg": 0.003762050997465849, + "policy/entropy_avg": 0.16833877563476562, + "step": 1148, + "val/clipfrac_avg": 0.010529030114412308, + "val/num_eos_tokens": 126, + "val/ratio": 0.9884514808654785, + "val/ratio_var": 3.90233144571539e-05 + }, + { + "episode": 18384, + "epoch": 0.9284848484848485, + "eps": 0, + "loss/policy_avg": -0.13550132513046265, + "loss/value_avg": 0.310035765171051, + "lr": 1.4381926513618139e-07, + "objective/entropy": -37.70539855957031, + "objective/kl": 41.92686462402344, + "objective/non_score_reward": -2.0963432788848877, + "objective/rlhf_reward": 14.622406959533691, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010944223031401634, + "policy/clipfrac_avg": 0.004749140702188015, + "policy/entropy_avg": 0.21954727172851562, + "step": 1149, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 114, + "val/ratio": 0.9851934313774109, + "val/ratio_var": 6.719871453242376e-05 + }, + { + "episode": 18400, + "epoch": 0.9292929292929293, + "eps": 0, + "loss/policy_avg": -0.06519678980112076, + "loss/value_avg": 0.4952123761177063, + "lr": 1.4065602363909747e-07, + "objective/entropy": -55.67281723022461, + "objective/kl": 38.189300537109375, + "objective/non_score_reward": -1.9094650745391846, + "objective/rlhf_reward": 14.028034210205078, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001011414104141295, + "policy/clipfrac_avg": 0.005307096056640148, + "policy/entropy_avg": 0.20238876342773438, + "step": 1150, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 103, + "val/ratio": 0.9863063097000122, + "val/ratio_var": 0.00010381833271821961 + }, + { + "episode": 18416, + "epoch": 0.9301010101010101, + "eps": 0, + "loss/policy_avg": 0.025855571031570435, + "loss/value_avg": 0.7742699980735779, + "lr": 1.375274608162447e-07, + "objective/entropy": -64.18814086914062, + "objective/kl": 39.29536056518555, + "objective/non_score_reward": -1.9647679328918457, + "objective/rlhf_reward": 13.746169090270996, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009069567313417792, + "policy/clipfrac_avg": 0.00267383293248713, + "policy/entropy_avg": 0.19156980514526367, + "step": 1151, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885040521621704, + "val/ratio_var": 7.799283775966614e-05 + }, + { + "episode": 18432, + "epoch": 0.9309090909090909, + "eps": 0, + "loss/policy_avg": 0.01431315392255783, + "loss/value_avg": 0.6286057829856873, + "lr": 1.3443359899458997e-07, + "objective/entropy": -59.235267639160156, + "objective/kl": 40.618900299072266, + "objective/non_score_reward": -2.030945062637329, + "objective/rlhf_reward": 11.54718017578125, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0008881621761247516, + "policy/clipfrac_avg": 0.0032237274572253227, + "policy/entropy_avg": 0.17874622344970703, + "step": 1152, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 353, + "val/ratio": 0.9887373447418213, + "val/ratio_var": 5.042510019848123e-05 + }, + { + "episode": 18448, + "epoch": 0.9317171717171717, + "eps": 0, + "loss/policy_avg": -0.0857156366109848, + "loss/value_avg": 0.2431882619857788, + "lr": 1.3137446025345557e-07, + "objective/entropy": -56.33277130126953, + "objective/kl": 46.32222366333008, + "objective/non_score_reward": -2.3161110877990723, + "objective/rlhf_reward": 15.996389389038086, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0009288631845265627, + "policy/clipfrac_avg": 0.0027455075178295374, + "policy/entropy_avg": 0.1957530975341797, + "step": 1153, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 26, + "val/ratio": 0.9869426488876343, + "val/ratio_var": 4.2594107071636245e-05 + }, + { + "episode": 18464, + "epoch": 0.9325252525252525, + "eps": 0, + "loss/policy_avg": -0.09154313802719116, + "loss/value_avg": 0.5833359360694885, + "lr": 1.283500664243642e-07, + "objective/entropy": -81.86225891113281, + "objective/kl": 33.21459197998047, + "objective/non_score_reward": -1.6607296466827393, + "objective/rlhf_reward": 13.62833309173584, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008894167840480804, + "policy/clipfrac_avg": 0.0032217884436249733, + "policy/entropy_avg": 0.16341471672058105, + "step": 1154, + "val/clipfrac_avg": 0.0005091222701594234, + "val/num_eos_tokens": 503, + "val/ratio": 0.9880057573318481, + "val/ratio_var": 7.092337909853086e-05 + }, + { + "episode": 18480, + "epoch": 0.9333333333333333, + "eps": 0, + "loss/policy_avg": 0.10637003928422928, + "loss/value_avg": 0.5488271713256836, + "lr": 1.253604390908819e-07, + "objective/entropy": -86.53608703613281, + "objective/kl": 37.38941192626953, + "objective/non_score_reward": -1.8694705963134766, + "objective/rlhf_reward": 14.591466903686523, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010367620270699263, + "policy/clipfrac_avg": 0.007691029459238052, + "policy/entropy_avg": 0.16927146911621094, + "step": 1155, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 399, + "val/ratio": 0.9881411790847778, + "val/ratio_var": 8.204267214750871e-05 + }, + { + "episode": 18496, + "epoch": 0.9341414141414142, + "eps": 0, + "loss/policy_avg": -0.024129033088684082, + "loss/value_avg": 0.4214504361152649, + "lr": 1.2240559958846266e-07, + "objective/entropy": -87.75643920898438, + "objective/kl": 44.4080810546875, + "objective/non_score_reward": -2.2204041481018066, + "objective/rlhf_reward": 14.404596328735352, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008544453303329647, + "policy/clipfrac_avg": 0.0035026988480240107, + "policy/entropy_avg": 0.16388320922851562, + "step": 1156, + "val/clipfrac_avg": 9.057971328729764e-05, + "val/num_eos_tokens": 8, + "val/ratio": 0.9889931082725525, + "val/ratio_var": 6.746850704075769e-05 + }, + { + "episode": 18512, + "epoch": 0.934949494949495, + "eps": 0, + "loss/policy_avg": -0.08957675099372864, + "loss/value_avg": 0.3355456590652466, + "lr": 1.1948556900429786e-07, + "objective/entropy": -90.10191345214844, + "objective/kl": 39.96202850341797, + "objective/non_score_reward": -1.9981014728546143, + "objective/rlhf_reward": 16.14252471923828, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008546580211259425, + "policy/clipfrac_avg": 0.005381043069064617, + "policy/entropy_avg": 0.16065406799316406, + "step": 1157, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 3, + "val/ratio": 0.989165186882019, + "val/ratio_var": 4.7844900109339505e-05 + }, + { + "episode": 18528, + "epoch": 0.9357575757575758, + "eps": 0, + "loss/policy_avg": -0.06473330408334732, + "loss/value_avg": 0.4225938320159912, + "lr": 1.1660036817716492e-07, + "objective/entropy": -63.458831787109375, + "objective/kl": 40.30827331542969, + "objective/non_score_reward": -2.015413761138916, + "objective/rlhf_reward": 13.633024215698242, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008652628748677671, + "policy/clipfrac_avg": 0.009679234586656094, + "policy/entropy_avg": 0.1683187484741211, + "step": 1158, + "val/clipfrac_avg": 0.000117481205961667, + "val/num_eos_tokens": 301, + "val/ratio": 0.989500880241394, + "val/ratio_var": 3.429090429563075e-05 + }, + { + "episode": 18544, + "epoch": 0.9365656565656566, + "eps": 0, + "loss/policy_avg": 0.19546493887901306, + "loss/value_avg": 0.6905479431152344, + "lr": 1.1375001769728e-07, + "objective/entropy": -65.75634765625, + "objective/kl": 36.45595169067383, + "objective/non_score_reward": -1.8227976560592651, + "objective/rlhf_reward": 13.138139724731445, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008474080823361874, + "policy/clipfrac_avg": 0.006815841421484947, + "policy/entropy_avg": 0.16717529296875, + "step": 1159, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 124, + "val/ratio": 0.9894546270370483, + "val/ratio_var": 7.423999340971932e-05 + }, + { + "episode": 18560, + "epoch": 0.9373737373737374, + "eps": 0, + "loss/policy_avg": 0.10427418351173401, + "loss/value_avg": 0.7115825414657593, + "lr": 1.1093453790614883e-07, + "objective/entropy": -168.40960693359375, + "objective/kl": 37.055477142333984, + "objective/non_score_reward": -1.852773666381836, + "objective/rlhf_reward": 14.084726333618164, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0007912840810604393, + "policy/clipfrac_avg": 0.0036162808537483215, + "policy/entropy_avg": 0.1563091278076172, + "step": 1160, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 427, + "val/ratio": 0.9901214838027954, + "val/ratio_var": 7.212928176159039e-05 + }, + { + "episode": 18576, + "epoch": 0.9381818181818182, + "eps": 0, + "loss/policy_avg": -0.12785540521144867, + "loss/value_avg": 0.39417845010757446, + "lr": 1.0815394889642339e-07, + "objective/entropy": -70.233154296875, + "objective/kl": 44.201019287109375, + "objective/non_score_reward": -2.2100508213043213, + "objective/rlhf_reward": 16.102449417114258, + "objective/scores": 18.25, + "policy/approxkl_avg": 0.0011408152058720589, + "policy/clipfrac_avg": 0.004944411106407642, + "policy/entropy_avg": 0.19538497924804688, + "step": 1161, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 331, + "val/ratio": 0.9865884184837341, + "val/ratio_var": 6.0507758462335914e-05 + }, + { + "episode": 18592, + "epoch": 0.938989898989899, + "eps": 0, + "loss/policy_avg": -0.03844344615936279, + "loss/value_avg": 0.655718207359314, + "lr": 1.0540827051175817e-07, + "objective/entropy": -37.648441314697266, + "objective/kl": 43.3171501159668, + "objective/non_score_reward": -2.1658573150634766, + "objective/rlhf_reward": 13.099767684936523, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009900203440338373, + "policy/clipfrac_avg": 0.003007909283041954, + "policy/entropy_avg": 0.20216941833496094, + "step": 1162, + "val/clipfrac_avg": 0.0003906250058207661, + "val/num_eos_tokens": 199, + "val/ratio": 0.9874058961868286, + "val/ratio_var": 6.270931771723554e-05 + }, + { + "episode": 18608, + "epoch": 0.9397979797979797, + "eps": 0, + "loss/policy_avg": 0.05490083992481232, + "loss/value_avg": 0.4446883201599121, + "lr": 1.0269752234666642e-07, + "objective/entropy": -83.78005981445312, + "objective/kl": 38.83243942260742, + "objective/non_score_reward": -1.941622018814087, + "objective/rlhf_reward": 15.863064765930176, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008689056267030537, + "policy/clipfrac_avg": 0.004589558579027653, + "policy/entropy_avg": 0.1755847930908203, + "step": 1163, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 209, + "val/ratio": 0.9878473281860352, + "val/ratio_var": 5.755285747000016e-05 + }, + { + "episode": 18624, + "epoch": 0.9406060606060606, + "eps": 0, + "loss/policy_avg": 0.5652229189872742, + "loss/value_avg": 4.454827785491943, + "lr": 1.0002172374638519e-07, + "objective/entropy": -106.71968078613281, + "objective/kl": 28.182273864746094, + "objective/non_score_reward": -1.4091136455535889, + "objective/rlhf_reward": 13.051824569702148, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006604374502785504, + "policy/clipfrac_avg": 0.006071992218494415, + "policy/entropy_avg": 0.13239336013793945, + "step": 1164, + "val/clipfrac_avg": 0.0032051282469183207, + "val/num_eos_tokens": 199, + "val/ratio": 0.9910687208175659, + "val/ratio_var": 5.329243867890909e-05 + }, + { + "episode": 18640, + "epoch": 0.9414141414141414, + "eps": 0, + "loss/policy_avg": 0.030554044991731644, + "loss/value_avg": 0.3813501298427582, + "lr": 9.738089380673e-08, + "objective/entropy": -50.39229965209961, + "objective/kl": 46.05052947998047, + "objective/non_score_reward": -2.3025264739990234, + "objective/rlhf_reward": 15.775598526000977, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0010013967985287309, + "policy/clipfrac_avg": 0.004134259186685085, + "policy/entropy_avg": 0.20552825927734375, + "step": 1165, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 147, + "val/ratio": 0.986208975315094, + "val/ratio_var": 5.7627206842880696e-05 + }, + { + "episode": 18656, + "epoch": 0.9422222222222222, + "eps": 0, + "loss/policy_avg": 0.014425485394895077, + "loss/value_avg": 0.17465442419052124, + "lr": 9.477505137396759e-08, + "objective/entropy": -48.5864143371582, + "objective/kl": 46.88139343261719, + "objective/non_score_reward": -2.344069719314575, + "objective/rlhf_reward": 14.749679565429688, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009588432731106877, + "policy/clipfrac_avg": 0.003158081788569689, + "policy/entropy_avg": 0.19722747802734375, + "step": 1166, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 150, + "val/ratio": 0.9872362613677979, + "val/ratio_var": 5.964138108538464e-05 + }, + { + "episode": 18672, + "epoch": 0.943030303030303, + "eps": 0, + "loss/policy_avg": 0.12084536254405975, + "loss/value_avg": 0.5831397771835327, + "lr": 9.22042150446728e-08, + "objective/entropy": -76.38983154296875, + "objective/kl": 38.01190948486328, + "objective/non_score_reward": -1.9005954265594482, + "objective/rlhf_reward": 11.919716835021973, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0007046982645988464, + "policy/clipfrac_avg": 0.0028958613984286785, + "policy/entropy_avg": 0.1511383056640625, + "step": 1167, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 417, + "val/ratio": 0.990734875202179, + "val/ratio_var": 6.113814743002877e-05 + }, + { + "episode": 18688, + "epoch": 0.9438383838383838, + "eps": 0, + "loss/policy_avg": -0.1721150279045105, + "loss/value_avg": 0.29965418577194214, + "lr": 8.966840316560255e-08, + "objective/entropy": -190.17677307128906, + "objective/kl": 44.538299560546875, + "objective/non_score_reward": -2.226915121078491, + "objective/rlhf_reward": 14.882460594177246, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0007411800906993449, + "policy/clipfrac_avg": 0.005143396090716124, + "policy/entropy_avg": 0.14809703826904297, + "step": 1168, + "val/clipfrac_avg": 1.3469827536027879e-05, + "val/num_eos_tokens": 76, + "val/ratio": 0.990103542804718, + "val/ratio_var": 4.799567614099942e-05 + }, + { + "episode": 18704, + "epoch": 0.9446464646464646, + "eps": 0, + "loss/policy_avg": -0.04706333950161934, + "loss/value_avg": 0.4846585988998413, + "lr": 8.716763383355863e-08, + "objective/entropy": -26.354211807250977, + "objective/kl": 43.76121520996094, + "objective/non_score_reward": -2.1880605220794678, + "objective/rlhf_reward": 12.905689239501953, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010139367077499628, + "policy/clipfrac_avg": 0.00270761433057487, + "policy/entropy_avg": 0.19493675231933594, + "step": 1169, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 515, + "val/ratio": 0.9869682788848877, + "val/ratio_var": 4.458573312149383e-05 + }, + { + "episode": 18720, + "epoch": 0.9454545454545454, + "eps": 0, + "loss/policy_avg": -0.11434232443571091, + "loss/value_avg": 0.32796037197113037, + "lr": 8.470192489526519e-08, + "objective/entropy": -118.44176483154297, + "objective/kl": 33.78557205200195, + "objective/non_score_reward": -1.6892786026000977, + "objective/rlhf_reward": 14.466971397399902, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008025875431485474, + "policy/clipfrac_avg": 0.005034630186855793, + "policy/entropy_avg": 0.17252540588378906, + "step": 1170, + "val/clipfrac_avg": 0.000240384615608491, + "val/num_eos_tokens": 534, + "val/ratio": 0.9892752766609192, + "val/ratio_var": 8.975582750281319e-05 + }, + { + "episode": 18736, + "epoch": 0.9462626262626262, + "eps": 0, + "loss/policy_avg": -0.26330432295799255, + "loss/value_avg": 0.43789923191070557, + "lr": 8.227129394723643e-08, + "objective/entropy": -90.45104217529297, + "objective/kl": 35.37781524658203, + "objective/non_score_reward": -1.7688908576965332, + "objective/rlhf_reward": 15.840483665466309, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0007532388553954661, + "policy/clipfrac_avg": 0.00782378576695919, + "policy/entropy_avg": 0.14811134338378906, + "step": 1171, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 449, + "val/ratio": 0.9901248216629028, + "val/ratio_var": 4.5044969738228247e-05 + }, + { + "episode": 18752, + "epoch": 0.9470707070707071, + "eps": 0, + "loss/policy_avg": 0.09692054986953735, + "loss/value_avg": 0.2597772777080536, + "lr": 7.987575833565298e-08, + "objective/entropy": -90.80900573730469, + "objective/kl": 45.11640930175781, + "objective/non_score_reward": -2.2558202743530273, + "objective/rlhf_reward": 13.744178771972656, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000919578829780221, + "policy/clipfrac_avg": 0.005172869190573692, + "policy/entropy_avg": 0.17873764038085938, + "step": 1172, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 400, + "val/ratio": 0.9880226850509644, + "val/ratio_var": 5.284639337332919e-05 + }, + { + "episode": 18768, + "epoch": 0.9478787878787879, + "eps": 0, + "loss/policy_avg": 0.09063289314508438, + "loss/value_avg": 0.5753982663154602, + "lr": 7.7515335156238e-08, + "objective/entropy": -163.6809844970703, + "objective/kl": 37.627655029296875, + "objective/non_score_reward": -1.8813828229904175, + "objective/rlhf_reward": 14.59518051147461, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0007881260244175792, + "policy/clipfrac_avg": 0.0037529754918068647, + "policy/entropy_avg": 0.15079402923583984, + "step": 1173, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 262, + "val/ratio": 0.9891691207885742, + "val/ratio_var": 4.976944182999432e-05 + }, + { + "episode": 18784, + "epoch": 0.9486868686868687, + "eps": 0, + "loss/policy_avg": 0.14138484001159668, + "loss/value_avg": 0.9088690280914307, + "lr": 7.519004125413454e-08, + "objective/entropy": -171.2418670654297, + "objective/kl": 38.040130615234375, + "objective/non_score_reward": -1.9020065069198608, + "objective/rlhf_reward": 13.176118850708008, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0006656637415289879, + "policy/clipfrac_avg": 0.0033770238514989614, + "policy/entropy_avg": 0.1490926742553711, + "step": 1174, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 584, + "val/ratio": 0.9910387992858887, + "val/ratio_var": 5.571794099523686e-05 + }, + { + "episode": 18800, + "epoch": 0.9494949494949495, + "eps": 0, + "loss/policy_avg": -0.09168200194835663, + "loss/value_avg": 0.44036585092544556, + "lr": 7.289989322378732e-08, + "objective/entropy": -50.938758850097656, + "objective/kl": 43.64716339111328, + "objective/non_score_reward": -2.1823582649230957, + "objective/rlhf_reward": 13.802017211914062, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009952617110684514, + "policy/clipfrac_avg": 0.0027715780306607485, + "policy/entropy_avg": 0.17894935607910156, + "step": 1175, + "val/clipfrac_avg": 0.0004111842135898769, + "val/num_eos_tokens": 169, + "val/ratio": 0.987088680267334, + "val/ratio_var": 6.252353341551498e-05 + }, + { + "episode": 18816, + "epoch": 0.9503030303030303, + "eps": 0, + "loss/policy_avg": -0.17822237312793732, + "loss/value_avg": 0.4382031559944153, + "lr": 7.064490740882057e-08, + "objective/entropy": -153.3197479248047, + "objective/kl": 43.219486236572266, + "objective/non_score_reward": -2.1609742641448975, + "objective/rlhf_reward": 12.741369247436523, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009114840067923069, + "policy/clipfrac_avg": 0.004199516959488392, + "policy/entropy_avg": 0.18891334533691406, + "step": 1176, + "val/clipfrac_avg": 3.814697265625e-05, + "val/num_eos_tokens": 23, + "val/ratio": 0.9868748784065247, + "val/ratio_var": 7.712288061156869e-05 + }, + { + "episode": 18832, + "epoch": 0.9511111111111111, + "eps": 0, + "loss/policy_avg": -0.29926538467407227, + "loss/value_avg": 0.24289564788341522, + "lr": 6.842509990192536e-08, + "objective/entropy": -62.70560073852539, + "objective/kl": 46.933937072753906, + "objective/non_score_reward": -2.3466968536376953, + "objective/rlhf_reward": 15.286115646362305, + "objective/scores": 17.625, + "policy/approxkl_avg": 0.0009256636258214712, + "policy/clipfrac_avg": 0.005345791578292847, + "policy/entropy_avg": 0.19078445434570312, + "step": 1177, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878906607627869, + "val/ratio_var": 5.454176425701007e-05 + }, + { + "episode": 18848, + "epoch": 0.9519191919191919, + "eps": 0, + "loss/policy_avg": -0.07624176144599915, + "loss/value_avg": 1.0276535749435425, + "lr": 6.624048654474247e-08, + "objective/entropy": -102.2330322265625, + "objective/kl": 38.56993103027344, + "objective/non_score_reward": -1.9284967184066772, + "objective/rlhf_reward": 13.352752685546875, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008128330809995532, + "policy/clipfrac_avg": 0.004654319025576115, + "policy/entropy_avg": 0.16016387939453125, + "step": 1178, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 33, + "val/ratio": 0.9896093010902405, + "val/ratio_var": 5.181640881346539e-05 + }, + { + "episode": 18864, + "epoch": 0.9527272727272728, + "eps": 0, + "loss/policy_avg": 0.035814136266708374, + "loss/value_avg": 0.5764297842979431, + "lr": 6.409108292774912e-08, + "objective/entropy": -59.69445037841797, + "objective/kl": 39.02704620361328, + "objective/non_score_reward": -1.9513523578643799, + "objective/rlhf_reward": 13.173646926879883, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001039088936522603, + "policy/clipfrac_avg": 0.003585438709706068, + "policy/entropy_avg": 0.19232654571533203, + "step": 1179, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 192, + "val/ratio": 0.9868942499160767, + "val/ratio_var": 6.958071753615513e-05 + }, + { + "episode": 18880, + "epoch": 0.9535353535353536, + "eps": 0, + "loss/policy_avg": 0.24797323346138, + "loss/value_avg": 0.2845846712589264, + "lr": 6.197690439015025e-08, + "objective/entropy": -40.500762939453125, + "objective/kl": 47.32844543457031, + "objective/non_score_reward": -2.366422176361084, + "objective/rlhf_reward": 13.852327346801758, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010082677472382784, + "policy/clipfrac_avg": 0.005784696899354458, + "policy/entropy_avg": 0.19600677490234375, + "step": 1180, + "val/clipfrac_avg": 0.00013241525448393077, + "val/num_eos_tokens": 504, + "val/ratio": 0.9869298934936523, + "val/ratio_var": 4.6343844587681815e-05 + }, + { + "episode": 18896, + "epoch": 0.9543434343434344, + "eps": 0, + "loss/policy_avg": -0.011056587100028992, + "loss/value_avg": 0.21195590496063232, + "lr": 5.989796601976516e-08, + "objective/entropy": -87.35359191894531, + "objective/kl": 38.190425872802734, + "objective/non_score_reward": -1.9095213413238525, + "objective/rlhf_reward": 12.574853897094727, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000693869951646775, + "policy/clipfrac_avg": 0.0022745702881366014, + "policy/entropy_avg": 0.14525604248046875, + "step": 1181, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 672, + "val/ratio": 0.9902079105377197, + "val/ratio_var": 5.0829687097575516e-05 + }, + { + "episode": 18912, + "epoch": 0.9551515151515152, + "eps": 0, + "loss/policy_avg": 0.5624556541442871, + "loss/value_avg": 0.936620831489563, + "lr": 5.785428265292381e-08, + "objective/entropy": -131.93319702148438, + "objective/kl": 36.28763961791992, + "objective/non_score_reward": -1.8143819570541382, + "objective/rlhf_reward": 12.099679946899414, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0006996375159360468, + "policy/clipfrac_avg": 0.0033093849197030067, + "policy/entropy_avg": 0.1407628059387207, + "step": 1182, + "val/clipfrac_avg": 0.0014880952658131719, + "val/num_eos_tokens": 240, + "val/ratio": 0.9909154176712036, + "val/ratio_var": 5.887184670427814e-05 + }, + { + "episode": 18928, + "epoch": 0.955959595959596, + "eps": 0, + "loss/policy_avg": -0.19112437963485718, + "loss/value_avg": 0.6247477531433105, + "lr": 5.584586887435739e-08, + "objective/entropy": -117.6522216796875, + "objective/kl": 39.66671371459961, + "objective/non_score_reward": -1.9833356142044067, + "objective/rlhf_reward": 13.297914505004883, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007932945154607296, + "policy/clipfrac_avg": 0.0029290905222296715, + "policy/entropy_avg": 0.16626358032226562, + "step": 1183, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 41, + "val/ratio": 0.9893531799316406, + "val/ratio_var": 5.722875357605517e-05 + }, + { + "episode": 18944, + "epoch": 0.9567676767676768, + "eps": 0, + "loss/policy_avg": -0.018872268497943878, + "loss/value_avg": 0.6814364194869995, + "lr": 5.387273901709733e-08, + "objective/entropy": -59.193870544433594, + "objective/kl": 38.85816955566406, + "objective/non_score_reward": -1.942908525466919, + "objective/rlhf_reward": 12.28365421295166, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009058800060302019, + "policy/clipfrac_avg": 0.004390705842524767, + "policy/entropy_avg": 0.17277050018310547, + "step": 1184, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 391, + "val/ratio": 0.9888806939125061, + "val/ratio_var": 5.870430322829634e-05 + }, + { + "episode": 18960, + "epoch": 0.9575757575757575, + "eps": 0, + "loss/policy_avg": -0.12193582952022552, + "loss/value_avg": 0.4072027802467346, + "lr": 5.1934907162370374e-08, + "objective/entropy": -72.95974731445312, + "objective/kl": 35.59428405761719, + "objective/non_score_reward": -1.7797143459320068, + "objective/rlhf_reward": 13.673410415649414, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0008201617165468633, + "policy/clipfrac_avg": 0.004145351238548756, + "policy/entropy_avg": 0.17438912391662598, + "step": 1185, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 60, + "val/ratio": 0.9888736605644226, + "val/ratio_var": 6.133368151495233e-05 + }, + { + "episode": 18976, + "epoch": 0.9583838383838383, + "eps": 0, + "loss/policy_avg": 0.3409832715988159, + "loss/value_avg": 2.1178839206695557, + "lr": 5.0032387139501425e-08, + "objective/entropy": -97.86048889160156, + "objective/kl": 41.09333038330078, + "objective/non_score_reward": -2.054666519165039, + "objective/rlhf_reward": 13.710959434509277, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.000780026544816792, + "policy/clipfrac_avg": 0.007402490358799696, + "policy/entropy_avg": 0.19989013671875, + "step": 1186, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 76, + "val/ratio": 0.9890344738960266, + "val/ratio_var": 7.444098446285352e-05 + }, + { + "episode": 18992, + "epoch": 0.9591919191919192, + "eps": 0, + "loss/policy_avg": -0.16587065160274506, + "loss/value_avg": 0.5858635902404785, + "lr": 4.8165192525809754e-08, + "objective/entropy": -31.46831512451172, + "objective/kl": 36.85295486450195, + "objective/non_score_reward": -1.842647671699524, + "objective/rlhf_reward": 14.922977447509766, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.001208727015182376, + "policy/clipfrac_avg": 0.004555144812911749, + "policy/entropy_avg": 0.20450401306152344, + "step": 1187, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 113, + "val/ratio": 0.9874365329742432, + "val/ratio_var": 5.309836706146598e-05 + }, + { + "episode": 19008, + "epoch": 0.96, + "eps": 0, + "loss/policy_avg": 0.06852813065052032, + "loss/value_avg": 0.32328563928604126, + "lr": 4.63333366465174e-08, + "objective/entropy": -105.82904815673828, + "objective/kl": 32.94627380371094, + "objective/non_score_reward": -1.6473137140274048, + "objective/rlhf_reward": 15.290185928344727, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0006628286791965365, + "policy/clipfrac_avg": 0.004865474533289671, + "policy/entropy_avg": 0.12561321258544922, + "step": 1188, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 202, + "val/ratio": 0.9917237758636475, + "val/ratio_var": 3.8588947063544765e-05 + }, + { + "episode": 19024, + "epoch": 0.9608080808080808, + "eps": 0, + "loss/policy_avg": 0.020303204655647278, + "loss/value_avg": 0.4330519437789917, + "lr": 4.4536832574650355e-08, + "objective/entropy": -33.98373031616211, + "objective/kl": 45.81948471069336, + "objective/non_score_reward": -2.2909743785858154, + "objective/rlhf_reward": 13.005900382995605, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010197311639785767, + "policy/clipfrac_avg": 0.003383245086297393, + "policy/entropy_avg": 0.21395492553710938, + "step": 1189, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 86, + "val/ratio": 0.985183835029602, + "val/ratio_var": 5.527482062461786e-05 + }, + { + "episode": 19040, + "epoch": 0.9616161616161616, + "eps": 0, + "loss/policy_avg": -0.6133148074150085, + "loss/value_avg": 0.3876040279865265, + "lr": 4.2775693130948094e-08, + "objective/entropy": -152.57052612304688, + "objective/kl": 38.252220153808594, + "objective/non_score_reward": -1.9126112461090088, + "objective/rlhf_reward": 14.89988899230957, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007638135575689375, + "policy/clipfrac_avg": 0.0035591432824730873, + "policy/entropy_avg": 0.16634774208068848, + "step": 1190, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 392, + "val/ratio": 0.9892970323562622, + "val/ratio_var": 6.732134352205321e-05 + }, + { + "episode": 19056, + "epoch": 0.9624242424242424, + "eps": 0, + "loss/policy_avg": -0.16719648241996765, + "loss/value_avg": 0.4944162368774414, + "lr": 4.104993088376974e-08, + "objective/entropy": -81.75244140625, + "objective/kl": 46.120994567871094, + "objective/non_score_reward": -2.3060498237609863, + "objective/rlhf_reward": 13.537699699401855, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009323159465566278, + "policy/clipfrac_avg": 0.003273975569754839, + "policy/entropy_avg": 0.19600319862365723, + "step": 1191, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 291, + "val/ratio": 0.9876390695571899, + "val/ratio_var": 4.03721714974381e-05 + }, + { + "episode": 19072, + "epoch": 0.9632323232323232, + "eps": 0, + "loss/policy_avg": 0.08641587942838669, + "loss/value_avg": 0.6021695137023926, + "lr": 3.935955814900527e-08, + "objective/entropy": -58.236507415771484, + "objective/kl": 38.58598327636719, + "objective/non_score_reward": -1.9292993545532227, + "objective/rlhf_reward": 13.148825645446777, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0010641806293278933, + "policy/clipfrac_avg": 0.004020705819129944, + "policy/entropy_avg": 0.19188880920410156, + "step": 1192, + "val/clipfrac_avg": 0.00021150591783225536, + "val/num_eos_tokens": 665, + "val/ratio": 0.9875043630599976, + "val/ratio_var": 7.918935443740338e-05 + }, + { + "episode": 19088, + "epoch": 0.964040404040404, + "eps": 0, + "loss/policy_avg": 0.09802606701850891, + "loss/value_avg": 1.0540528297424316, + "lr": 3.77045869899878e-08, + "objective/entropy": -126.37530517578125, + "objective/kl": 36.086280822753906, + "objective/non_score_reward": -1.8043140172958374, + "objective/rlhf_reward": 13.906623840332031, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008227983489632607, + "policy/clipfrac_avg": 0.004573613405227661, + "policy/entropy_avg": 0.15534448623657227, + "step": 1193, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 325, + "val/ratio": 0.9889199733734131, + "val/ratio_var": 9.045511978911236e-05 + }, + { + "episode": 19104, + "epoch": 0.9648484848484848, + "eps": 0, + "loss/policy_avg": -0.17688646912574768, + "loss/value_avg": 0.4370724558830261, + "lr": 3.608502921740753e-08, + "objective/entropy": -35.11339569091797, + "objective/kl": 32.869319915771484, + "objective/non_score_reward": -1.6434661149978638, + "objective/rlhf_reward": 14.551846504211426, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.002314617158845067, + "policy/clipfrac_avg": 0.0031052969861775637, + "policy/entropy_avg": 0.17627334594726562, + "step": 1194, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 185, + "val/ratio": 0.989388108253479, + "val/ratio_var": 5.49936194147449e-05 + }, + { + "episode": 19120, + "epoch": 0.9656565656565657, + "eps": 0, + "loss/policy_avg": -0.17504875361919403, + "loss/value_avg": 0.204596608877182, + "lr": 3.450089638922738e-08, + "objective/entropy": -92.06233978271484, + "objective/kl": 40.820533752441406, + "objective/non_score_reward": -2.0410265922546387, + "objective/rlhf_reward": 14.513660430908203, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009747116710059345, + "policy/clipfrac_avg": 0.005887703038752079, + "policy/entropy_avg": 0.17911338806152344, + "step": 1195, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 294, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 4.2761934309965e-05 + }, + { + "episode": 19136, + "epoch": 0.9664646464646465, + "eps": 0, + "loss/policy_avg": -0.12304341048002243, + "loss/value_avg": 0.32093346118927, + "lr": 3.2952199810599714e-08, + "objective/entropy": -24.155014038085938, + "objective/kl": 45.62751007080078, + "objective/non_score_reward": -2.2813756465911865, + "objective/rlhf_reward": 14.195186614990234, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010361537570133805, + "policy/clipfrac_avg": 0.006471775472164154, + "policy/entropy_avg": 0.19795608520507812, + "step": 1196, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 209, + "val/ratio": 0.9868389368057251, + "val/ratio_var": 5.5822583817644045e-05 + }, + { + "episode": 19152, + "epoch": 0.9672727272727273, + "eps": 0, + "loss/policy_avg": 0.3443071246147156, + "loss/value_avg": 1.5372172594070435, + "lr": 3.143895053378698e-08, + "objective/entropy": -111.23834228515625, + "objective/kl": 31.232709884643555, + "objective/non_score_reward": -1.5616354942321777, + "objective/rlhf_reward": 13.35242748260498, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0007176597137004137, + "policy/clipfrac_avg": 0.004162969067692757, + "policy/entropy_avg": 0.2422410249710083, + "step": 1197, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 524, + "val/ratio": 0.9907166957855225, + "val/ratio_var": 7.462361827492714e-05 + }, + { + "episode": 19168, + "epoch": 0.9680808080808081, + "eps": 0, + "loss/policy_avg": -0.13895125687122345, + "loss/value_avg": 0.5284525752067566, + "lr": 2.9961159358082305e-08, + "objective/entropy": -101.44007110595703, + "objective/kl": 32.114402770996094, + "objective/non_score_reward": -1.6057199239730835, + "objective/rlhf_reward": 14.378654479980469, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.000817152438685298, + "policy/clipfrac_avg": 0.004995419643819332, + "policy/entropy_avg": 0.15488815307617188, + "step": 1198, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 443, + "val/ratio": 0.9890648126602173, + "val/ratio_var": 7.766807539155707e-05 + }, + { + "episode": 19184, + "epoch": 0.9688888888888889, + "eps": 0, + "loss/policy_avg": -0.2223341315984726, + "loss/value_avg": 0.6924841403961182, + "lr": 2.8518836829732332e-08, + "objective/entropy": -71.81754302978516, + "objective/kl": 37.39568328857422, + "objective/non_score_reward": -1.8697842359542847, + "objective/rlhf_reward": 13.520840644836426, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008745577652007341, + "policy/clipfrac_avg": 0.002502672839909792, + "policy/entropy_avg": 0.18534183502197266, + "step": 1199, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 599, + "val/ratio": 0.9876028299331665, + "val/ratio_var": 5.801619045087136e-05 + }, + { + "episode": 19200, + "epoch": 0.9696969696969697, + "eps": 0, + "loss/policy_avg": 0.15578943490982056, + "loss/value_avg": 0.9613189101219177, + "lr": 2.7111993241860646e-08, + "objective/entropy": -74.75447082519531, + "objective/kl": 31.942792892456055, + "objective/non_score_reward": -1.597139596939087, + "objective/rlhf_reward": 13.043485641479492, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007055438472889364, + "policy/clipfrac_avg": 0.0035820305347442627, + "policy/entropy_avg": 0.15447449684143066, + "step": 1200, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 539, + "val/ratio": 0.9913029670715332, + "val/ratio_var": 5.023244739277288e-05 + }, + { + "episode": 19216, + "epoch": 0.9705050505050505, + "eps": 0, + "loss/policy_avg": -0.1969611942768097, + "loss/value_avg": 0.46793222427368164, + "lr": 2.5740638634397243e-08, + "objective/entropy": -63.07292175292969, + "objective/kl": 37.90913391113281, + "objective/non_score_reward": -1.8954565525054932, + "objective/rlhf_reward": 14.112356185913086, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008319137268699706, + "policy/clipfrac_avg": 0.0048570772632956505, + "policy/entropy_avg": 0.15787792205810547, + "step": 1201, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 176, + "val/ratio": 0.9898661375045776, + "val/ratio_var": 3.5913133615395054e-05 + }, + { + "episode": 19232, + "epoch": 0.9713131313131314, + "eps": 0, + "loss/policy_avg": -0.0762750431895256, + "loss/value_avg": 0.7102093696594238, + "lr": 2.440478279400471e-08, + "objective/entropy": -85.2213134765625, + "objective/kl": 42.020835876464844, + "objective/non_score_reward": -2.101041793823242, + "objective/rlhf_reward": 14.586458206176758, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009388537146151066, + "policy/clipfrac_avg": 0.004320736508816481, + "policy/entropy_avg": 0.18203043937683105, + "step": 1202, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 108, + "val/ratio": 0.9878751635551453, + "val/ratio_var": 8.284598879981786e-05 + }, + { + "episode": 19248, + "epoch": 0.9721212121212122, + "eps": 0, + "loss/policy_avg": 0.023966491222381592, + "loss/value_avg": 0.36540549993515015, + "lr": 2.3104435254008852e-08, + "objective/entropy": -50.760467529296875, + "objective/kl": 33.10966491699219, + "objective/non_score_reward": -1.6554831266403198, + "objective/rlhf_reward": 14.21170425415039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010394756682217121, + "policy/clipfrac_avg": 0.005734467413276434, + "policy/entropy_avg": 0.18365859985351562, + "step": 1203, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 456, + "val/ratio": 0.9874485731124878, + "val/ratio_var": 6.813264917582273e-05 + }, + { + "episode": 19264, + "epoch": 0.972929292929293, + "eps": 0, + "loss/policy_avg": 0.11596876382827759, + "loss/value_avg": 0.8759022951126099, + "lr": 2.1839605294330935e-08, + "objective/entropy": -41.6898307800293, + "objective/kl": 34.464874267578125, + "objective/non_score_reward": -1.7232437133789062, + "objective/rlhf_reward": 14.409568786621094, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008781763026490808, + "policy/clipfrac_avg": 0.0046472325921058655, + "policy/entropy_avg": 0.18979644775390625, + "step": 1204, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 8, + "val/ratio": 0.988035261631012, + "val/ratio_var": 7.465987437171862e-05 + }, + { + "episode": 19280, + "epoch": 0.9737373737373738, + "eps": 0, + "loss/policy_avg": 0.033114898949861526, + "loss/value_avg": 0.37955981492996216, + "lr": 2.0610301941420552e-08, + "objective/entropy": -64.19001770019531, + "objective/kl": 38.246883392333984, + "objective/non_score_reward": -1.912344217300415, + "objective/rlhf_reward": 12.517343521118164, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008579955901950598, + "policy/clipfrac_avg": 0.002436618786305189, + "policy/entropy_avg": 0.18629741668701172, + "step": 1205, + "val/clipfrac_avg": 0.006591796875, + "val/num_eos_tokens": 791, + "val/ratio": 0.9880963563919067, + "val/ratio_var": 3.743668276001699e-05 + }, + { + "episode": 19296, + "epoch": 0.9745454545454545, + "eps": 0, + "loss/policy_avg": -0.37829723954200745, + "loss/value_avg": 0.47165149450302124, + "lr": 1.9416533968193428e-08, + "objective/entropy": -47.87005615234375, + "objective/kl": 38.28486633300781, + "objective/non_score_reward": -1.9142433404922485, + "objective/rlhf_reward": 15.296693801879883, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010683179134503007, + "policy/clipfrac_avg": 0.004513275809586048, + "policy/entropy_avg": 0.2166946828365326, + "step": 1206, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 24, + "val/ratio": 0.9851523041725159, + "val/ratio_var": 8.785770478425547e-05 + }, + { + "episode": 19312, + "epoch": 0.9753535353535353, + "eps": 0, + "loss/policy_avg": -0.12085376679897308, + "loss/value_avg": 0.5372148156166077, + "lr": 1.8258309893965375e-08, + "objective/entropy": -79.00741577148438, + "objective/kl": 45.99113464355469, + "objective/non_score_reward": -2.2995567321777344, + "objective/rlhf_reward": 13.708255767822266, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009223736124113202, + "policy/clipfrac_avg": 0.0040555731393396854, + "policy/entropy_avg": 0.17890167236328125, + "step": 1207, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 75, + "val/ratio": 0.9878267049789429, + "val/ratio_var": 6.0516715166158974e-05 + }, + { + "episode": 19328, + "epoch": 0.9761616161616161, + "eps": 0, + "loss/policy_avg": -0.08923263102769852, + "loss/value_avg": 0.5102915167808533, + "lr": 1.713563798439455e-08, + "objective/entropy": -66.35103607177734, + "objective/kl": 42.04967498779297, + "objective/non_score_reward": -2.1024839878082275, + "objective/rlhf_reward": 14.780328750610352, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009762570261955261, + "policy/clipfrac_avg": 0.004208541475236416, + "policy/entropy_avg": 0.19926166534423828, + "step": 1208, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 505, + "val/ratio": 0.987288236618042, + "val/ratio_var": 6.778948591090739e-05 + }, + { + "episode": 19344, + "epoch": 0.9769696969696969, + "eps": 0, + "loss/policy_avg": 0.10433784127235413, + "loss/value_avg": 0.8428288698196411, + "lr": 1.6048526251421502e-08, + "objective/entropy": -112.43741607666016, + "objective/kl": 42.122718811035156, + "objective/non_score_reward": -2.106135845184326, + "objective/rlhf_reward": 11.198551177978516, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009249926079064608, + "policy/clipfrac_avg": 0.004424647428095341, + "policy/entropy_avg": 0.19216203689575195, + "step": 1209, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 27, + "val/ratio": 0.9879051446914673, + "val/ratio_var": 6.320722604868934e-05 + }, + { + "episode": 19360, + "epoch": 0.9777777777777777, + "eps": 0, + "loss/policy_avg": 0.16060814261436462, + "loss/value_avg": 0.33783212304115295, + "lr": 1.4996982453211463e-08, + "objective/entropy": -77.9478530883789, + "objective/kl": 34.459083557128906, + "objective/non_score_reward": -1.722954273223877, + "objective/rlhf_reward": 14.105171203613281, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.001020944444462657, + "policy/clipfrac_avg": 0.004141935613006353, + "policy/entropy_avg": 0.18428802490234375, + "step": 1210, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 65, + "val/ratio": 0.9876895546913147, + "val/ratio_var": 0.00010520015348447487 + }, + { + "episode": 19376, + "epoch": 0.9785858585858586, + "eps": 0, + "loss/policy_avg": -0.06701843440532684, + "loss/value_avg": 0.3668726682662964, + "lr": 1.3981014094099354e-08, + "objective/entropy": -46.33203125, + "objective/kl": 35.14643096923828, + "objective/non_score_reward": -1.7573217153549194, + "objective/rlhf_reward": 13.656740188598633, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009761269902810454, + "policy/clipfrac_avg": 0.003752158023416996, + "policy/entropy_avg": 0.19565200805664062, + "step": 1211, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 46, + "val/ratio": 0.9875230193138123, + "val/ratio_var": 6.482725439127535e-05 + }, + { + "episode": 19392, + "epoch": 0.9793939393939394, + "eps": 0, + "loss/policy_avg": -0.2390209585428238, + "loss/value_avg": 0.7244433760643005, + "lr": 1.3000628424535978e-08, + "objective/entropy": -69.5890884399414, + "objective/kl": 28.53516387939453, + "objective/non_score_reward": -1.426758050918579, + "objective/rlhf_reward": 14.3076171875, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0006685786065645516, + "policy/clipfrac_avg": 0.0029422249644994736, + "policy/entropy_avg": 0.16259002685546875, + "step": 1212, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 143, + "val/ratio": 0.9912441968917847, + "val/ratio_var": 4.31347252742853e-05 + }, + { + "episode": 19408, + "epoch": 0.9802020202020202, + "eps": 0, + "loss/policy_avg": -0.11228986084461212, + "loss/value_avg": 0.5388200283050537, + "lr": 1.2055832441036918e-08, + "objective/entropy": -82.7415542602539, + "objective/kl": 41.615177154541016, + "objective/non_score_reward": -2.080759048461914, + "objective/rlhf_reward": 14.606740951538086, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009461919544264674, + "policy/clipfrac_avg": 0.006980390287935734, + "policy/entropy_avg": 0.18620681762695312, + "step": 1213, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 134, + "val/ratio": 0.987679123878479, + "val/ratio_var": 6.926806236151606e-05 + }, + { + "episode": 19424, + "epoch": 0.981010101010101, + "eps": 0, + "loss/policy_avg": -0.20061983168125153, + "loss/value_avg": 0.2869341969490051, + "lr": 1.1146632886132603e-08, + "objective/entropy": -104.67575073242188, + "objective/kl": 38.947208404541016, + "objective/non_score_reward": -1.9473603963851929, + "objective/rlhf_reward": 14.27138900756836, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0007925564423203468, + "policy/clipfrac_avg": 0.0025324940215796232, + "policy/entropy_avg": 0.15836048126220703, + "step": 1214, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 256, + "val/ratio": 0.9894617795944214, + "val/ratio_var": 4.7174267820082605e-05 + }, + { + "episode": 19440, + "epoch": 0.9818181818181818, + "eps": 0, + "loss/policy_avg": -0.12940247356891632, + "loss/value_avg": 0.5062823295593262, + "lr": 1.0273036248318325e-08, + "objective/entropy": -101.33221435546875, + "objective/kl": 41.90412521362305, + "objective/non_score_reward": -2.0952064990997314, + "objective/rlhf_reward": 14.529792785644531, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008722887723706663, + "policy/clipfrac_avg": 0.0025138547644019127, + "policy/entropy_avg": 0.17874455451965332, + "step": 1215, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 248, + "val/ratio": 0.9877287149429321, + "val/ratio_var": 6.869715434731916e-05 + }, + { + "episode": 19456, + "epoch": 0.9826262626262626, + "eps": 0, + "loss/policy_avg": -0.39164668321609497, + "loss/value_avg": 1.550127387046814, + "lr": 9.435048762010957e-09, + "objective/entropy": -192.84432983398438, + "objective/kl": 35.63471984863281, + "objective/non_score_reward": -1.781736135482788, + "objective/rlhf_reward": 11.92138957977295, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009429135243408382, + "policy/clipfrac_avg": 0.0026213740929961205, + "policy/entropy_avg": 0.17586326599121094, + "step": 1216, + "val/clipfrac_avg": 7.62939453125e-06, + "val/num_eos_tokens": 61, + "val/ratio": 0.9886311888694763, + "val/ratio_var": 7.116834603948519e-05 + }, + { + "episode": 19472, + "epoch": 0.9834343434343434, + "eps": 0, + "loss/policy_avg": 0.08832584321498871, + "loss/value_avg": 0.4303143620491028, + "lr": 8.632676407502316e-09, + "objective/entropy": -42.98954391479492, + "objective/kl": 39.298187255859375, + "objective/non_score_reward": -1.964909315109253, + "objective/rlhf_reward": 14.472591400146484, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009566507651470602, + "policy/clipfrac_avg": 0.004461781587451696, + "policy/entropy_avg": 0.18021583557128906, + "step": 1217, + "val/clipfrac_avg": 0.001953125, + "val/num_eos_tokens": 243, + "val/ratio": 0.9864565134048462, + "val/ratio_var": 3.8086534914327785e-05 + }, + { + "episode": 19488, + "epoch": 0.9842424242424243, + "eps": 0, + "loss/policy_avg": 0.21013334393501282, + "loss/value_avg": 0.4151275157928467, + "lr": 7.865924910916977e-09, + "objective/entropy": -107.35826110839844, + "objective/kl": 37.92512512207031, + "objective/non_score_reward": -1.896256446838379, + "objective/rlhf_reward": 13.080305099487305, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009007140179164708, + "policy/clipfrac_avg": 0.0045885005965828896, + "policy/entropy_avg": 0.18993210792541504, + "step": 1218, + "val/clipfrac_avg": 0.0007440476329065859, + "val/num_eos_tokens": 164, + "val/ratio": 0.9882104396820068, + "val/ratio_var": 7.511641160817817e-05 + }, + { + "episode": 19504, + "epoch": 0.9850505050505051, + "eps": 0, + "loss/policy_avg": 0.1631181836128235, + "loss/value_avg": 0.3172687888145447, + "lr": 7.13479974417175e-09, + "objective/entropy": -103.62552642822266, + "objective/kl": 42.538116455078125, + "objective/non_score_reward": -2.126905918121338, + "objective/rlhf_reward": 13.982468605041504, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008123687002807856, + "policy/clipfrac_avg": 0.004256902262568474, + "policy/entropy_avg": 0.16496849060058594, + "step": 1219, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 67, + "val/ratio": 0.9897924661636353, + "val/ratio_var": 3.71673668269068e-05 + }, + { + "episode": 19520, + "epoch": 0.9858585858585859, + "eps": 0, + "loss/policy_avg": -0.18715566396713257, + "loss/value_avg": 0.5615487098693848, + "lr": 6.439306124936262e-09, + "objective/entropy": -120.69778442382812, + "objective/kl": 44.76129150390625, + "objective/non_score_reward": -2.2380645275115967, + "objective/rlhf_reward": 14.80099868774414, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0009488088544458151, + "policy/clipfrac_avg": 0.0036685834638774395, + "policy/entropy_avg": 0.17595529556274414, + "step": 1220, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 949, + "val/ratio": 0.9875380992889404, + "val/ratio_var": 7.448292308254167e-05 + }, + { + "episode": 19536, + "epoch": 0.9866666666666667, + "eps": 0, + "loss/policy_avg": -0.4247804880142212, + "loss/value_avg": 0.9726248979568481, + "lr": 5.779449016595773e-09, + "objective/entropy": -120.10693359375, + "objective/kl": 42.00151062011719, + "objective/non_score_reward": -2.1000754833221436, + "objective/rlhf_reward": 14.212424278259277, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008749752887524664, + "policy/clipfrac_avg": 0.004988180007785559, + "policy/entropy_avg": 0.1530447006225586, + "step": 1221, + "val/clipfrac_avg": 0.00023134966613724828, + "val/num_eos_tokens": 39, + "val/ratio": 0.9896517992019653, + "val/ratio_var": 5.084315125714056e-05 + }, + { + "episode": 19552, + "epoch": 0.9874747474747475, + "eps": 0, + "loss/policy_avg": 0.17928610742092133, + "loss/value_avg": 0.8346552848815918, + "lr": 5.155233128216197e-09, + "objective/entropy": -129.04684448242188, + "objective/kl": 36.87531280517578, + "objective/non_score_reward": -1.8437657356262207, + "objective/rlhf_reward": 13.230453491210938, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0007634559297002852, + "policy/clipfrac_avg": 0.0055130645632743835, + "policy/entropy_avg": 0.14986801147460938, + "step": 1222, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 563, + "val/ratio": 0.9903774857521057, + "val/ratio_var": 5.780787250841968e-05 + }, + { + "episode": 19568, + "epoch": 0.9882828282828283, + "eps": 0, + "loss/policy_avg": -0.028081759810447693, + "loss/value_avg": 0.3895527124404907, + "lr": 4.56666291450858e-09, + "objective/entropy": -49.08431625366211, + "objective/kl": 39.414207458496094, + "objective/non_score_reward": -1.9707105159759521, + "objective/rlhf_reward": 13.771476745605469, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0010028551332652569, + "policy/clipfrac_avg": 0.005500209983438253, + "policy/entropy_avg": 0.2012653350830078, + "step": 1223, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 38, + "val/ratio": 0.987224280834198, + "val/ratio_var": 4.987093052477576e-05 + }, + { + "episode": 19584, + "epoch": 0.9890909090909091, + "eps": 0, + "loss/policy_avg": -0.014931255020201206, + "loss/value_avg": 0.4837211072444916, + "lr": 4.0137425758018935e-09, + "objective/entropy": -40.7100830078125, + "objective/kl": 37.389583587646484, + "objective/non_score_reward": -1.8694791793823242, + "objective/rlhf_reward": 13.505520820617676, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001194866606965661, + "policy/clipfrac_avg": 0.005203597247600555, + "policy/entropy_avg": 0.22698020935058594, + "step": 1224, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 375, + "val/ratio": 0.9855931401252747, + "val/ratio_var": 7.236239616759121e-05 + }, + { + "episode": 19600, + "epoch": 0.98989898989899, + "eps": 0, + "loss/policy_avg": 0.05002763122320175, + "loss/value_avg": 0.33766478300094604, + "lr": 3.496476058006959e-09, + "objective/entropy": -75.71792602539062, + "objective/kl": 40.458518981933594, + "objective/non_score_reward": -2.022926092147827, + "objective/rlhf_reward": 14.328636169433594, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009783877758309245, + "policy/clipfrac_avg": 0.002967006526887417, + "policy/entropy_avg": 0.194549560546875, + "step": 1225, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863731265068054, + "val/ratio_var": 4.958164936397225e-05 + }, + { + "episode": 19616, + "epoch": 0.9907070707070708, + "eps": 0, + "loss/policy_avg": -0.09883704781532288, + "loss/value_avg": 0.40289461612701416, + "lr": 3.014867052593684e-09, + "objective/entropy": -77.89369201660156, + "objective/kl": 45.668174743652344, + "objective/non_score_reward": -2.2834086418151855, + "objective/rlhf_reward": 14.654090881347656, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.000869191309902817, + "policy/clipfrac_avg": 0.0027626235969364643, + "policy/entropy_avg": 0.18253517150878906, + "step": 1226, + "val/clipfrac_avg": 0.0002861721732188016, + "val/num_eos_tokens": 48, + "val/ratio": 0.9884761571884155, + "val/ratio_var": 5.125232564751059e-05 + }, + { + "episode": 19632, + "epoch": 0.9915151515151515, + "eps": 0, + "loss/policy_avg": 0.02128005027770996, + "loss/value_avg": 0.34802740812301636, + "lr": 2.568918996560532e-09, + "objective/entropy": -133.7528839111328, + "objective/kl": 41.73491668701172, + "objective/non_score_reward": -2.0867457389831543, + "objective/rlhf_reward": 13.960128784179688, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007204288267530501, + "policy/clipfrac_avg": 0.00297390203922987, + "policy/entropy_avg": 0.14430570602416992, + "step": 1227, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 316, + "val/ratio": 0.9903041124343872, + "val/ratio_var": 6.99873999110423e-05 + }, + { + "episode": 19648, + "epoch": 0.9923232323232323, + "eps": 0, + "loss/policy_avg": -0.30528533458709717, + "loss/value_avg": 0.7530245780944824, + "lr": 2.1586350724134285e-09, + "objective/entropy": -44.44089126586914, + "objective/kl": 42.037235260009766, + "objective/non_score_reward": -2.1018619537353516, + "objective/rlhf_reward": 14.179388046264648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009157764725387096, + "policy/clipfrac_avg": 0.003242512932047248, + "policy/entropy_avg": 0.19690704345703125, + "step": 1228, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 429, + "val/ratio": 0.9876857995986938, + "val/ratio_var": 5.984380186419003e-05 + }, + { + "episode": 19664, + "epoch": 0.9931313131313131, + "eps": 0, + "loss/policy_avg": 0.07529310882091522, + "loss/value_avg": 0.36114734411239624, + "lr": 1.7840182081396707e-09, + "objective/entropy": -50.87998962402344, + "objective/kl": 43.7442741394043, + "objective/non_score_reward": -2.187213897705078, + "objective/rlhf_reward": 15.812786102294922, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0009651206783019006, + "policy/clipfrac_avg": 0.003591865533962846, + "policy/entropy_avg": 0.19472503662109375, + "step": 1229, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 331, + "val/ratio": 0.9876270294189453, + "val/ratio_var": 4.56350389868021e-05 + }, + { + "episode": 19680, + "epoch": 0.9939393939393939, + "eps": 0, + "loss/policy_avg": -0.08426038920879364, + "loss/value_avg": 0.3430386185646057, + "lr": 1.4450710771907184e-09, + "objective/entropy": -69.65153503417969, + "objective/kl": 38.04791259765625, + "objective/non_score_reward": -1.9023958444595337, + "objective/rlhf_reward": 13.425728797912598, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.000832535675726831, + "policy/clipfrac_avg": 0.003121623769402504, + "policy/entropy_avg": 0.17328572273254395, + "step": 1230, + "val/clipfrac_avg": 0.0010775862028822303, + "val/num_eos_tokens": 327, + "val/ratio": 0.9891126155853271, + "val/ratio_var": 5.81457388761919e-05 + }, + { + "episode": 19696, + "epoch": 0.9947474747474747, + "eps": 0, + "loss/policy_avg": 0.002345144748687744, + "loss/value_avg": 1.1214191913604736, + "lr": 1.1417960984605459e-09, + "objective/entropy": -27.681867599487305, + "objective/kl": 36.740516662597656, + "objective/non_score_reward": -1.8370258808135986, + "objective/rlhf_reward": 13.741098403930664, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009004836902022362, + "policy/clipfrac_avg": 0.005135446786880493, + "policy/entropy_avg": 0.18401336669921875, + "step": 1231, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 137, + "val/ratio": 0.9884154796600342, + "val/ratio_var": 7.396236469503492e-05 + }, + { + "episode": 19712, + "epoch": 0.9955555555555555, + "eps": 0, + "loss/policy_avg": -0.16311845183372498, + "loss/value_avg": 0.45753657817840576, + "lr": 8.741954362678773e-10, + "objective/entropy": -45.30944061279297, + "objective/kl": 45.00796890258789, + "objective/non_score_reward": -2.250398635864258, + "objective/rlhf_reward": 15.554288864135742, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0009604179067537189, + "policy/clipfrac_avg": 0.004513026215136051, + "policy/entropy_avg": 0.18356657028198242, + "step": 1232, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889092445373535, + "val/ratio_var": 4.4623157009482384e-05 + }, + { + "episode": 19728, + "epoch": 0.9963636363636363, + "eps": 0, + "loss/policy_avg": -0.2625172734260559, + "loss/value_avg": 0.8098385334014893, + "lr": 6.422710003439747e-10, + "objective/entropy": -19.211952209472656, + "objective/kl": 37.37506103515625, + "objective/non_score_reward": -1.8687530755996704, + "objective/rlhf_reward": 12.959371566772461, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.000893141608685255, + "policy/clipfrac_avg": 0.003839832032099366, + "policy/entropy_avg": 0.1701374053955078, + "step": 1233, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 12, + "val/ratio": 0.9891498684883118, + "val/ratio_var": 7.125152478693053e-05 + }, + { + "episode": 19744, + "epoch": 0.9971717171717172, + "eps": 0, + "loss/policy_avg": 0.10235726833343506, + "loss/value_avg": 0.41452595591545105, + "lr": 4.4602444581542945e-10, + "objective/entropy": -53.28070831298828, + "objective/kl": 46.30189895629883, + "objective/non_score_reward": -2.3150949478149414, + "objective/rlhf_reward": 14.263029098510742, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.001009408151730895, + "policy/clipfrac_avg": 0.006422281265258789, + "policy/entropy_avg": 0.19713115692138672, + "step": 1234, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 349, + "val/ratio": 0.9869934320449829, + "val/ratio_var": 9.03628533706069e-05 + }, + { + "episode": 19760, + "epoch": 0.997979797979798, + "eps": 0, + "loss/policy_avg": -0.13045719265937805, + "loss/value_avg": 0.8233026266098022, + "lr": 2.854571731947253e-10, + "objective/entropy": -68.4776611328125, + "objective/kl": 37.48680877685547, + "objective/non_score_reward": -1.874340534210205, + "objective/rlhf_reward": 14.492847442626953, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009436773834750056, + "policy/clipfrac_avg": 0.005421591456979513, + "policy/entropy_avg": 0.1775188446044922, + "step": 1235, + "val/clipfrac_avg": 4.183400233159773e-05, + "val/num_eos_tokens": 52, + "val/ratio": 0.988511860370636, + "val/ratio_var": 5.036559741711244e-05 + }, + { + "episode": 19776, + "epoch": 0.9987878787878788, + "eps": 0, + "loss/policy_avg": -0.21915020048618317, + "loss/value_avg": 0.6580561995506287, + "lr": 1.6057032836913623e-10, + "objective/entropy": -155.7568359375, + "objective/kl": 44.79389190673828, + "objective/non_score_reward": -2.239694595336914, + "objective/rlhf_reward": 14.119680404663086, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007429021643474698, + "policy/clipfrac_avg": 0.0029837298206984997, + "policy/entropy_avg": 0.14424514770507812, + "step": 1236, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 405, + "val/ratio": 0.9901254773139954, + "val/ratio_var": 6.483635661425069e-05 + }, + { + "episode": 19792, + "epoch": 0.9995959595959596, + "eps": 0, + "loss/policy_avg": 0.19540390372276306, + "loss/value_avg": 1.0495953559875488, + "lr": 7.136480259239964e-11, + "objective/entropy": -55.40583801269531, + "objective/kl": 38.987640380859375, + "objective/non_score_reward": -1.9493823051452637, + "objective/rlhf_reward": 12.292805671691895, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008245522622019053, + "policy/clipfrac_avg": 0.002606890397146344, + "policy/entropy_avg": 0.16810131072998047, + "step": 1237, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 134, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 4.211545819998719e-05 + }, + { + "episode": 19808, + "epoch": 1.0004040404040404, + "eps": 0, + "loss/policy_avg": 0.025396723300218582, + "loss/value_avg": 0.5493187308311462, + "lr": 1.784123247916547e-11, + "objective/entropy": -112.83091735839844, + "objective/kl": 39.11707305908203, + "objective/non_score_reward": -1.9558537006378174, + "objective/rlhf_reward": 11.758989334106445, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0008375335019081831, + "policy/clipfrac_avg": 0.004157694987952709, + "policy/entropy_avg": 0.15214157104492188, + "step": 1238, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 299, + "val/ratio": 0.9900014400482178, + "val/ratio_var": 6.577932072104886e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-1238/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18b9e56f00f6677eeec75673f6adca45ca6ca820 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d039dd9637544a9a319311ecef9c10d9899f17c8577c24fd5024fcd8a6be1f95 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7cfec1255a27cab7023f89dc8b179ee6076acc25 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/trainer_state.json @@ -0,0 +1,4235 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 3200, + "epoch": 0.16161616161616163, + "eval_steps": 100, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-200/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2b65e4d19e9a2d528fbf518c121435a7caebae03 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43cd52639380332bcd18698afd3949a1a84b6da6d7515a3f35cdf94b3d7fc15 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..dfee73634769449049ba228b5cc9828c53a41e7c --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/trainer_state.json @@ -0,0 +1,6335 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 4800, + "epoch": 0.24242424242424243, + "eval_steps": 100, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-300/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-300/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8bb5a6dd2329d48e56482bb40215355872836b58 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c4d77f6110cc85ee30ca7f06580c7d5c9347dfd54984be16c2b9206fb9d79d8 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..02efac9e411063364bf1e81529af63378ff605cd --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/trainer_state.json @@ -0,0 +1,8435 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 6400, + "epoch": 0.32323232323232326, + "eval_steps": 100, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-400/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-400/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..137dcb9cf653cbd912a85d0b20f66c8f147b81a8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e02c9b859583f51c4d62e927315b0cae1697768609b75d3d1d09d82d8769c6c4 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..598c7308432a050d1b246af9b4418631fb892917 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/trainer_state.json @@ -0,0 +1,10535 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 8000, + "epoch": 0.40404040404040403, + "eval_steps": 100, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-500/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..35726ed84d588c8aa3ef169859fb520780d77e22 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef61fbd275fc07586aefb437708b6fa9138cc5ae2fe6adca94a2ab859b8b723 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2b37fa26147095e7c633cca226efac5f5d656ada --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/trainer_state.json @@ -0,0 +1,12635 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 9600, + "epoch": 0.48484848484848486, + "eval_steps": 100, + "global_step": 600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-600/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9ab71b4279ec431c69a291f430e7160a8a2ee4ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15f06b4ab669eb2c66f82f1fb4632f65f29545a8737d4689fbb300021c9e3cf7 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5ebadb99140b56c0dbea5435557e23be934b90bc --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/trainer_state.json @@ -0,0 +1,14735 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 11200, + "epoch": 0.5656565656565656, + "eval_steps": 100, + "global_step": 700, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-700/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-700/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7f8965ffc8e136f37eaffc9d4e5e9cf082fdf3e2 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f76ac9abeb05bb52e0248ef55cbcf8f6066eaec2d2e7891ef3f66655130a42ef +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e24910f119b05ecb631d8c39d61d2bc25a6492d6 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/trainer_state.json @@ -0,0 +1,16835 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 12800, + "epoch": 0.6464646464646465, + "eval_steps": 100, + "global_step": 800, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + }, + { + "episode": 11216, + "epoch": 0.5664646464646464, + "eps": 0, + "loss/policy_avg": -0.09055965393781662, + "loss/value_avg": 0.5455071926116943, + "lr": 4.334128543718295e-06, + "objective/entropy": -74.95745849609375, + "objective/kl": 33.42132568359375, + "objective/non_score_reward": -1.6710662841796875, + "objective/rlhf_reward": 14.828933715820312, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0013184514828026295, + "policy/clipfrac_avg": 0.00809475127607584, + "policy/entropy_avg": 0.17902374267578125, + "step": 701, + "val/clipfrac_avg": 0.18833743035793304, + "val/num_eos_tokens": 358, + "val/ratio": 0.9879262447357178, + "val/ratio_var": 6.120221951277927e-05 + }, + { + "episode": 11232, + "epoch": 0.5672727272727273, + "eps": 0, + "loss/policy_avg": 0.2261451780796051, + "loss/value_avg": 1.5403046607971191, + "lr": 4.320892800523827e-06, + "objective/entropy": -90.61648559570312, + "objective/kl": 27.537485122680664, + "objective/non_score_reward": -1.3768744468688965, + "objective/rlhf_reward": 12.826250076293945, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009536159923300147, + "policy/clipfrac_avg": 0.01545755285769701, + "policy/entropy_avg": 0.20562219619750977, + "step": 702, + "val/clipfrac_avg": 0.033450622111558914, + "val/num_eos_tokens": 595, + "val/ratio": 0.9873440265655518, + "val/ratio_var": 7.613677007611841e-05 + }, + { + "episode": 11248, + "epoch": 0.5680808080808081, + "eps": 0, + "loss/policy_avg": -0.05153674632310867, + "loss/value_avg": 0.18954747915267944, + "lr": 4.307661903773129e-06, + "objective/entropy": -56.119659423828125, + "objective/kl": 43.216548919677734, + "objective/non_score_reward": -2.160827398300171, + "objective/rlhf_reward": 15.80792236328125, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010682737920433283, + "policy/clipfrac_avg": 0.005114596337080002, + "policy/entropy_avg": 0.20781707763671875, + "step": 703, + "val/clipfrac_avg": 0.12178023159503937, + "val/num_eos_tokens": 244, + "val/ratio": 0.9860707521438599, + "val/ratio_var": 5.392924140323885e-05 + }, + { + "episode": 11264, + "epoch": 0.5688888888888889, + "eps": 0, + "loss/policy_avg": 0.22105136513710022, + "loss/value_avg": 0.43452760577201843, + "lr": 4.294435947888405e-06, + "objective/entropy": -180.00924682617188, + "objective/kl": 32.27890396118164, + "objective/non_score_reward": -1.6139451265335083, + "objective/rlhf_reward": 12.425117492675781, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007389815291389823, + "policy/clipfrac_avg": 0.00613048393279314, + "policy/entropy_avg": 0.15887287259101868, + "step": 704, + "val/clipfrac_avg": 0.12273523956537247, + "val/num_eos_tokens": 611, + "val/ratio": 0.9900252819061279, + "val/ratio_var": 6.185931124491617e-05 + }, + { + "episode": 11280, + "epoch": 0.5696969696969697, + "eps": 0, + "loss/policy_avg": -0.044020622968673706, + "loss/value_avg": 1.459969401359558, + "lr": 4.281215027256592e-06, + "objective/entropy": -73.2503890991211, + "objective/kl": 29.207317352294922, + "objective/non_score_reward": -1.460365891456604, + "objective/rlhf_reward": 15.766195297241211, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010228066239506006, + "policy/clipfrac_avg": 0.010731351561844349, + "policy/entropy_avg": 0.17015647888183594, + "step": 705, + "val/clipfrac_avg": 0.1418685019016266, + "val/num_eos_tokens": 378, + "val/ratio": 0.9865360856056213, + "val/ratio_var": 0.0001155980207840912 + }, + { + "episode": 11296, + "epoch": 0.5705050505050505, + "eps": 0, + "loss/policy_avg": 0.10734500735998154, + "loss/value_avg": 0.40052735805511475, + "lr": 4.2679992362287e-06, + "objective/entropy": -91.05632019042969, + "objective/kl": 39.329261779785156, + "objective/non_score_reward": -1.966463327407837, + "objective/rlhf_reward": 12.744474411010742, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010600072564557195, + "policy/clipfrac_avg": 0.005786027293652296, + "policy/entropy_avg": 0.23768234252929688, + "step": 706, + "val/clipfrac_avg": 0.032500892877578735, + "val/num_eos_tokens": 203, + "val/ratio": 0.9860942959785461, + "val/ratio_var": 5.682241317117587e-05 + }, + { + "episode": 11312, + "epoch": 0.5713131313131313, + "eps": 0, + "loss/policy_avg": -0.5062837600708008, + "loss/value_avg": 0.7742470502853394, + "lr": 4.254788669119127e-06, + "objective/entropy": -93.2535400390625, + "objective/kl": 34.540130615234375, + "objective/non_score_reward": -1.727006435394287, + "objective/rlhf_reward": 14.788618087768555, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008475214708596468, + "policy/clipfrac_avg": 0.027025574818253517, + "policy/entropy_avg": 0.21597671508789062, + "step": 707, + "val/clipfrac_avg": 0.1775030940771103, + "val/num_eos_tokens": 165, + "val/ratio": 0.9894334077835083, + "val/ratio_var": 9.222733206115663e-05 + }, + { + "episode": 11328, + "epoch": 0.5721212121212121, + "eps": 0, + "loss/policy_avg": -0.1463669091463089, + "loss/value_avg": 0.5632686614990234, + "lr": 4.241583420204998e-06, + "objective/entropy": -110.37503051757812, + "objective/kl": 36.62909698486328, + "objective/non_score_reward": -1.831454873085022, + "objective/rlhf_reward": 12.89510726928711, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009251610608771443, + "policy/clipfrac_avg": 0.00941617414355278, + "policy/entropy_avg": 0.16801166534423828, + "step": 708, + "val/clipfrac_avg": 0.04514690116047859, + "val/num_eos_tokens": 267, + "val/ratio": 0.9897239208221436, + "val/ratio_var": 7.692311191931367e-05 + }, + { + "episode": 11344, + "epoch": 0.572929292929293, + "eps": 0, + "loss/policy_avg": -0.35295569896698, + "loss/value_avg": 0.7035583257675171, + "lr": 4.228383583725471e-06, + "objective/entropy": -76.27204132080078, + "objective/kl": 31.900005340576172, + "objective/non_score_reward": -1.5950002670288086, + "objective/rlhf_reward": 14.053436279296875, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010020419722422957, + "policy/clipfrac_avg": 0.011507621966302395, + "policy/entropy_avg": 0.16324424743652344, + "step": 709, + "val/clipfrac_avg": 0.12508229911327362, + "val/num_eos_tokens": 173, + "val/ratio": 0.9896982908248901, + "val/ratio_var": 4.9723090342013165e-05 + }, + { + "episode": 11360, + "epoch": 0.5737373737373738, + "eps": 0, + "loss/policy_avg": 0.160404771566391, + "loss/value_avg": 1.1914963722229004, + "lr": 4.215189253881093e-06, + "objective/entropy": -79.55085754394531, + "objective/kl": 36.81852722167969, + "objective/non_score_reward": -1.8409266471862793, + "objective/rlhf_reward": 12.159073829650879, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0011080782860517502, + "policy/clipfrac_avg": 0.006503967102617025, + "policy/entropy_avg": 0.19612884521484375, + "step": 710, + "val/clipfrac_avg": 0.08140315115451813, + "val/num_eos_tokens": 538, + "val/ratio": 0.9866147041320801, + "val/ratio_var": 7.869482942624018e-05 + }, + { + "episode": 11376, + "epoch": 0.5745454545454546, + "eps": 0, + "loss/policy_avg": 0.07548795640468597, + "loss/value_avg": 0.5138013362884521, + "lr": 4.2020005248331056e-06, + "objective/entropy": -119.03164672851562, + "objective/kl": 34.31198501586914, + "objective/non_score_reward": -1.7155992984771729, + "objective/rlhf_reward": 13.503150939941406, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007362160831689835, + "policy/clipfrac_avg": 0.00589254405349493, + "policy/entropy_avg": 0.14674091339111328, + "step": 711, + "val/clipfrac_avg": 0.02668866142630577, + "val/num_eos_tokens": 515, + "val/ratio": 0.990146279335022, + "val/ratio_var": 3.9673428545938805e-05 + }, + { + "episode": 11392, + "epoch": 0.5753535353535354, + "eps": 0, + "loss/policy_avg": -0.2925986051559448, + "loss/value_avg": 0.3821415603160858, + "lr": 4.1888174907027775e-06, + "objective/entropy": -112.06816101074219, + "objective/kl": 38.006500244140625, + "objective/non_score_reward": -1.900325059890747, + "objective/rlhf_reward": 13.990300178527832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.000874843040946871, + "policy/clipfrac_avg": 0.007544378750026226, + "policy/entropy_avg": 0.16875195503234863, + "step": 712, + "val/clipfrac_avg": 0.025153353810310364, + "val/num_eos_tokens": 4, + "val/ratio": 0.9893897771835327, + "val/ratio_var": 8.519255788996816e-05 + }, + { + "episode": 11408, + "epoch": 0.5761616161616162, + "eps": 0, + "loss/policy_avg": -0.5444605350494385, + "loss/value_avg": 1.517474889755249, + "lr": 4.175640245570742e-06, + "objective/entropy": -92.28114318847656, + "objective/kl": 35.69511413574219, + "objective/non_score_reward": -1.7847557067871094, + "objective/rlhf_reward": 13.73868179321289, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008441185345873237, + "policy/clipfrac_avg": 0.005670716054737568, + "policy/entropy_avg": 0.1591033935546875, + "step": 713, + "val/clipfrac_avg": 0.1791025698184967, + "val/num_eos_tokens": 98, + "val/ratio": 0.9903852939605713, + "val/ratio_var": 4.043114677187987e-05 + }, + { + "episode": 11424, + "epoch": 0.576969696969697, + "eps": 0, + "loss/policy_avg": -0.3059786260128021, + "loss/value_avg": 0.6795393228530884, + "lr": 4.162468883476319e-06, + "objective/entropy": -193.77468872070312, + "objective/kl": 42.17440414428711, + "objective/non_score_reward": -2.108720302581787, + "objective/rlhf_reward": 12.281905174255371, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008298080647364259, + "policy/clipfrac_avg": 0.00635810662060976, + "policy/entropy_avg": 0.15823698043823242, + "step": 714, + "val/clipfrac_avg": 0.0064142923802137375, + "val/num_eos_tokens": 1666, + "val/ratio": 0.9892975091934204, + "val/ratio_var": 6.228621350601315e-05 + }, + { + "episode": 11440, + "epoch": 0.5777777777777777, + "eps": 0, + "loss/policy_avg": -0.023257479071617126, + "loss/value_avg": 0.18199321627616882, + "lr": 4.149303498416838e-06, + "objective/entropy": -22.63348388671875, + "objective/kl": 44.308345794677734, + "objective/non_score_reward": -2.2154173851013184, + "objective/rlhf_reward": 14.82364559173584, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0012546201469376683, + "policy/clipfrac_avg": 0.007735411170870066, + "policy/entropy_avg": 0.240936279296875, + "step": 715, + "val/clipfrac_avg": 0.0028473599813878536, + "val/num_eos_tokens": 302, + "val/ratio": 0.9840776920318604, + "val/ratio_var": 5.949061960563995e-05 + }, + { + "episode": 11456, + "epoch": 0.5785858585858585, + "eps": 0, + "loss/policy_avg": 0.3237575888633728, + "loss/value_avg": 0.38197559118270874, + "lr": 4.136144184346979e-06, + "objective/entropy": -168.4746856689453, + "objective/kl": 40.63113784790039, + "objective/non_score_reward": -2.031557083129883, + "objective/rlhf_reward": 13.070005416870117, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0007934231543913484, + "policy/clipfrac_avg": 0.012050900608301163, + "policy/entropy_avg": 0.14413070678710938, + "step": 716, + "val/clipfrac_avg": 0.09322839975357056, + "val/num_eos_tokens": 1174, + "val/ratio": 0.9900047779083252, + "val/ratio_var": 5.29003118572291e-05 + }, + { + "episode": 11472, + "epoch": 0.5793939393939394, + "eps": 0, + "loss/policy_avg": 0.23654325306415558, + "loss/value_avg": 1.166879653930664, + "lr": 4.122991035178093e-06, + "objective/entropy": -96.77055358886719, + "objective/kl": 42.17526626586914, + "objective/non_score_reward": -2.1087632179260254, + "objective/rlhf_reward": 14.500611305236816, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000815618084743619, + "policy/clipfrac_avg": 0.008080050349235535, + "policy/entropy_avg": 0.14806270599365234, + "step": 717, + "val/clipfrac_avg": 0.02973807230591774, + "val/num_eos_tokens": 297, + "val/ratio": 0.9899397492408752, + "val/ratio_var": 3.871042645187117e-05 + }, + { + "episode": 11488, + "epoch": 0.5802020202020202, + "eps": 0, + "loss/policy_avg": -0.11968575417995453, + "loss/value_avg": 0.9426589608192444, + "lr": 4.109844144777537e-06, + "objective/entropy": -58.205047607421875, + "objective/kl": 42.6419792175293, + "objective/non_score_reward": -2.132099151611328, + "objective/rlhf_reward": 12.203838348388672, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009566791704855859, + "policy/clipfrac_avg": 0.0025268401950597763, + "policy/entropy_avg": 0.18893003463745117, + "step": 718, + "val/clipfrac_avg": 0.1098824143409729, + "val/num_eos_tokens": 16, + "val/ratio": 0.9869214296340942, + "val/ratio_var": 8.384165266761556e-05 + }, + { + "episode": 11504, + "epoch": 0.581010101010101, + "eps": 0, + "loss/policy_avg": 0.0208362378180027, + "loss/value_avg": 0.8499202728271484, + "lr": 4.096703606968007e-06, + "objective/entropy": -105.71292114257812, + "objective/kl": 40.70527648925781, + "objective/non_score_reward": -2.035263776779175, + "objective/rlhf_reward": 13.605360984802246, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008132618968375027, + "policy/clipfrac_avg": 0.004117591306567192, + "policy/entropy_avg": 0.15980243682861328, + "step": 719, + "val/clipfrac_avg": 0.04368842393159866, + "val/num_eos_tokens": 428, + "val/ratio": 0.9895676374435425, + "val/ratio_var": 6.342094275169075e-05 + }, + { + "episode": 11520, + "epoch": 0.5818181818181818, + "eps": 0, + "loss/policy_avg": -0.11627452820539474, + "loss/value_avg": 0.7368168830871582, + "lr": 4.083569515526853e-06, + "objective/entropy": -100.23733520507812, + "objective/kl": 32.93721008300781, + "objective/non_score_reward": -1.6468604803085327, + "objective/rlhf_reward": 15.134389877319336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008385346736758947, + "policy/clipfrac_avg": 0.009182492271065712, + "policy/entropy_avg": 0.17890548706054688, + "step": 720, + "val/clipfrac_avg": 0.06867726147174835, + "val/num_eos_tokens": 144, + "val/ratio": 0.9896779656410217, + "val/ratio_var": 7.192800694610924e-05 + }, + { + "episode": 11536, + "epoch": 0.5826262626262626, + "eps": 0, + "loss/policy_avg": -0.06581735610961914, + "loss/value_avg": 0.7933100461959839, + "lr": 4.070441964185428e-06, + "objective/entropy": -103.3985366821289, + "objective/kl": 39.985321044921875, + "objective/non_score_reward": -1.9992659091949463, + "objective/rlhf_reward": 14.547609329223633, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000881620217114687, + "policy/clipfrac_avg": 0.004334163852035999, + "policy/entropy_avg": 0.1855926513671875, + "step": 721, + "val/clipfrac_avg": 0.04114583134651184, + "val/num_eos_tokens": 161, + "val/ratio": 0.989449679851532, + "val/ratio_var": 6.215622852323577e-05 + }, + { + "episode": 11552, + "epoch": 0.5834343434343434, + "eps": 0, + "loss/policy_avg": 0.44041353464126587, + "loss/value_avg": 0.7198789119720459, + "lr": 4.05732104662841e-06, + "objective/entropy": -119.77040100097656, + "objective/kl": 36.42585754394531, + "objective/non_score_reward": -1.8212929964065552, + "objective/rlhf_reward": 13.467769622802734, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008070225012488663, + "policy/clipfrac_avg": 0.007022568956017494, + "policy/entropy_avg": 0.15318524837493896, + "step": 722, + "val/clipfrac_avg": 0.05163312703371048, + "val/num_eos_tokens": 304, + "val/ratio": 0.9899815320968628, + "val/ratio_var": 4.889031333732419e-05 + }, + { + "episode": 11568, + "epoch": 0.5842424242424242, + "eps": 0, + "loss/policy_avg": 0.4866091012954712, + "loss/value_avg": 1.1953067779541016, + "lr": 4.04420685649314e-06, + "objective/entropy": -50.46599578857422, + "objective/kl": 39.113037109375, + "objective/non_score_reward": -1.955651879310608, + "objective/rlhf_reward": 13.911535263061523, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010366745991632342, + "policy/clipfrac_avg": 0.006084335036575794, + "policy/entropy_avg": 0.20149445533752441, + "step": 723, + "val/clipfrac_avg": 0.09489050507545471, + "val/num_eos_tokens": 36, + "val/ratio": 0.9867548942565918, + "val/ratio_var": 6.938407750567421e-05 + }, + { + "episode": 11584, + "epoch": 0.585050505050505, + "eps": 0, + "loss/policy_avg": 0.1589544266462326, + "loss/value_avg": 0.568170964717865, + "lr": 4.031099487368937e-06, + "objective/entropy": -112.03081512451172, + "objective/kl": 37.355255126953125, + "objective/non_score_reward": -1.8677629232406616, + "objective/rlhf_reward": 13.49161148071289, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001035548048093915, + "policy/clipfrac_avg": 0.006865138188004494, + "policy/entropy_avg": 0.20564031600952148, + "step": 724, + "val/clipfrac_avg": 0.05233292281627655, + "val/num_eos_tokens": 347, + "val/ratio": 0.9859846830368042, + "val/ratio_var": 0.00010150864545721561 + }, + { + "episode": 11600, + "epoch": 0.5858585858585859, + "eps": 0, + "loss/policy_avg": -0.19635428488254547, + "loss/value_avg": 0.33023789525032043, + "lr": 4.0179990327964526e-06, + "objective/entropy": -69.79931640625, + "objective/kl": 41.90481948852539, + "objective/non_score_reward": -2.095241069793701, + "objective/rlhf_reward": 14.85007095336914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0010360007872805, + "policy/clipfrac_avg": 0.004560987465083599, + "policy/entropy_avg": 0.20290184020996094, + "step": 725, + "val/clipfrac_avg": 0.08375490456819534, + "val/num_eos_tokens": 261, + "val/ratio": 0.98649001121521, + "val/ratio_var": 6.110723916208372e-05 + }, + { + "episode": 11616, + "epoch": 0.5866666666666667, + "eps": 0, + "loss/policy_avg": 0.0999169647693634, + "loss/value_avg": 1.0625457763671875, + "lr": 4.004905586266988e-06, + "objective/entropy": -104.75285339355469, + "objective/kl": 32.01334762573242, + "objective/non_score_reward": -1.6006674766540527, + "objective/rlhf_reward": 12.766520500183105, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010034965816885233, + "policy/clipfrac_avg": 0.007322699762880802, + "policy/entropy_avg": 0.19988441467285156, + "step": 726, + "val/clipfrac_avg": 0.14562752842903137, + "val/num_eos_tokens": 33, + "val/ratio": 0.9887305498123169, + "val/ratio_var": 7.558859942946583e-05 + }, + { + "episode": 11632, + "epoch": 0.5874747474747475, + "eps": 0, + "loss/policy_avg": 0.004431502893567085, + "loss/value_avg": 0.5651569366455078, + "lr": 3.991819241221836e-06, + "objective/entropy": -53.65510940551758, + "objective/kl": 43.23419952392578, + "objective/non_score_reward": -2.161710262298584, + "objective/rlhf_reward": 12.986727714538574, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001010584644973278, + "policy/clipfrac_avg": 0.006680442951619625, + "policy/entropy_avg": 0.17978286743164062, + "step": 727, + "val/clipfrac_avg": 0.028638603165745735, + "val/num_eos_tokens": 25, + "val/ratio": 0.9880765676498413, + "val/ratio_var": 3.508182999212295e-05 + }, + { + "episode": 11648, + "epoch": 0.5882828282828283, + "eps": 0, + "loss/policy_avg": -0.026804931461811066, + "loss/value_avg": 0.3243325352668762, + "lr": 3.978740091051599e-06, + "objective/entropy": -147.60580444335938, + "objective/kl": 40.000404357910156, + "objective/non_score_reward": -2.0000202655792236, + "objective/rlhf_reward": 13.304666519165039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008220034651458263, + "policy/clipfrac_avg": 0.010559421963989735, + "policy/entropy_avg": 0.15192937850952148, + "step": 728, + "val/clipfrac_avg": 0.047058261930942535, + "val/num_eos_tokens": 93, + "val/ratio": 0.9890267252922058, + "val/ratio_var": 7.159873348427936e-05 + }, + { + "episode": 11664, + "epoch": 0.5890909090909091, + "eps": 0, + "loss/policy_avg": 0.061702877283096313, + "loss/value_avg": 0.25130608677864075, + "lr": 3.965668229095546e-06, + "objective/entropy": -75.26506042480469, + "objective/kl": 34.32489013671875, + "objective/non_score_reward": -1.7162445783615112, + "objective/rlhf_reward": 13.9556303024292, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008753975853323936, + "policy/clipfrac_avg": 0.006253514904528856, + "policy/entropy_avg": 0.17108154296875, + "step": 729, + "val/clipfrac_avg": 0.03564208000898361, + "val/num_eos_tokens": 194, + "val/ratio": 0.9883392453193665, + "val/ratio_var": 5.439678716356866e-05 + }, + { + "episode": 11680, + "epoch": 0.5898989898989899, + "eps": 0, + "loss/policy_avg": 0.5118429064750671, + "loss/value_avg": 0.3809865713119507, + "lr": 3.9526037486409275e-06, + "objective/entropy": -211.5462646484375, + "objective/kl": 38.28752136230469, + "objective/non_score_reward": -1.9143760204315186, + "objective/rlhf_reward": 13.062187194824219, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008200224256142974, + "policy/clipfrac_avg": 0.009272422641515732, + "policy/entropy_avg": 0.15618228912353516, + "step": 730, + "val/clipfrac_avg": 0.13058164715766907, + "val/num_eos_tokens": 191, + "val/ratio": 0.9892928600311279, + "val/ratio_var": 5.910802065045573e-05 + }, + { + "episode": 11696, + "epoch": 0.5907070707070707, + "eps": 0, + "loss/policy_avg": -0.575583279132843, + "loss/value_avg": 0.6588011384010315, + "lr": 3.939546742922318e-06, + "objective/entropy": -155.975341796875, + "objective/kl": 33.536251068115234, + "objective/non_score_reward": -1.6768126487731934, + "objective/rlhf_reward": 14.198186874389648, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006373925716616213, + "policy/clipfrac_avg": 0.011547036468982697, + "policy/entropy_avg": 0.15375041961669922, + "step": 731, + "val/clipfrac_avg": 0.1309666782617569, + "val/num_eos_tokens": 393, + "val/ratio": 0.9912899732589722, + "val/ratio_var": 7.494341116398573e-05 + }, + { + "episode": 11712, + "epoch": 0.5915151515151515, + "eps": 0, + "loss/policy_avg": 0.08975643664598465, + "loss/value_avg": 0.4319133758544922, + "lr": 3.926497305120943e-06, + "objective/entropy": -46.366912841796875, + "objective/kl": 38.771400451660156, + "objective/non_score_reward": -1.9385700225830078, + "objective/rlhf_reward": 14.631742477416992, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010276802349835634, + "policy/clipfrac_avg": 0.004419120494276285, + "policy/entropy_avg": 0.19952774047851562, + "step": 732, + "val/clipfrac_avg": 0.02601846680045128, + "val/num_eos_tokens": 121, + "val/ratio": 0.9856929779052734, + "val/ratio_var": 5.2173218136886135e-05 + }, + { + "episode": 11728, + "epoch": 0.5923232323232324, + "eps": 0, + "loss/policy_avg": -0.5671601295471191, + "loss/value_avg": 1.4009994268417358, + "lr": 3.913455528364027e-06, + "objective/entropy": -30.65240478515625, + "objective/kl": 34.644386291503906, + "objective/non_score_reward": -1.7322194576263428, + "objective/rlhf_reward": 16.127155303955078, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009229708230122924, + "policy/clipfrac_avg": 0.0053136469796299934, + "policy/entropy_avg": 0.1684565544128418, + "step": 733, + "val/clipfrac_avg": 0.16779303550720215, + "val/num_eos_tokens": 257, + "val/ratio": 0.98807692527771, + "val/ratio_var": 4.935808829031885e-05 + }, + { + "episode": 11744, + "epoch": 0.5931313131313132, + "eps": 0, + "loss/policy_avg": 0.1117314025759697, + "loss/value_avg": 0.6027213335037231, + "lr": 3.900421505724119e-06, + "objective/entropy": -115.830078125, + "objective/kl": 38.92997741699219, + "objective/non_score_reward": -1.9464988708496094, + "objective/rlhf_reward": 12.38162612915039, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006995117873884737, + "policy/clipfrac_avg": 0.0067988173104822636, + "policy/entropy_avg": 0.12786293029785156, + "step": 734, + "val/clipfrac_avg": 0.11902365833520889, + "val/num_eos_tokens": 256, + "val/ratio": 0.9913206696510315, + "val/ratio_var": 3.536290387273766e-05 + }, + { + "episode": 11760, + "epoch": 0.593939393939394, + "eps": 0, + "loss/policy_avg": 0.07707646489143372, + "loss/value_avg": 0.8557188510894775, + "lr": 3.887395330218429e-06, + "objective/entropy": -77.05795288085938, + "objective/kl": 34.79220199584961, + "objective/non_score_reward": -1.739610195159912, + "objective/rlhf_reward": 12.510390281677246, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009019911522045732, + "policy/clipfrac_avg": 0.00438590906560421, + "policy/entropy_avg": 0.18432998657226562, + "step": 735, + "val/clipfrac_avg": 0.20910227298736572, + "val/num_eos_tokens": 339, + "val/ratio": 0.9879709482192993, + "val/ratio_var": 7.31311592971906e-05 + }, + { + "episode": 11776, + "epoch": 0.5947474747474748, + "eps": 0, + "loss/policy_avg": 0.005743637681007385, + "loss/value_avg": 0.4823000729084015, + "lr": 3.8743770948081675e-06, + "objective/entropy": -189.58908081054688, + "objective/kl": 40.491024017333984, + "objective/non_score_reward": -2.0245513916015625, + "objective/rlhf_reward": 12.608261108398438, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010324950562790036, + "policy/clipfrac_avg": 0.008821255527436733, + "policy/entropy_avg": 0.1864786148071289, + "step": 736, + "val/clipfrac_avg": 0.03615584969520569, + "val/num_eos_tokens": 146, + "val/ratio": 0.9867680072784424, + "val/ratio_var": 0.00012203787628095597 + }, + { + "episode": 11792, + "epoch": 0.5955555555555555, + "eps": 0, + "loss/policy_avg": 0.11549818515777588, + "loss/value_avg": 0.4191131591796875, + "lr": 3.861366892397878e-06, + "objective/entropy": -53.57340621948242, + "objective/kl": 47.2681999206543, + "objective/non_score_reward": -2.363409996032715, + "objective/rlhf_reward": 11.238153457641602, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0010416920995339751, + "policy/clipfrac_avg": 0.006328020244836807, + "policy/entropy_avg": 0.21573257446289062, + "step": 737, + "val/clipfrac_avg": 0.05454040318727493, + "val/num_eos_tokens": 145, + "val/ratio": 0.9863517880439758, + "val/ratio_var": 5.17384396516718e-05 + }, + { + "episode": 11808, + "epoch": 0.5963636363636363, + "eps": 0, + "loss/policy_avg": 0.09252296388149261, + "loss/value_avg": 0.24368464946746826, + "lr": 3.848364815834782e-06, + "objective/entropy": -9.551445007324219, + "objective/kl": 41.81903839111328, + "objective/non_score_reward": -2.090951919555664, + "objective/rlhf_reward": 15.362173080444336, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.001077502267435193, + "policy/clipfrac_avg": 0.007244379259645939, + "policy/entropy_avg": 0.208465576171875, + "step": 738, + "val/clipfrac_avg": 0.046727634966373444, + "val/num_eos_tokens": 70, + "val/ratio": 0.9859708547592163, + "val/ratio_var": 4.752089807880111e-05 + }, + { + "episode": 11824, + "epoch": 0.5971717171717171, + "eps": 0, + "loss/policy_avg": -0.47144564986228943, + "loss/value_avg": 0.9659825563430786, + "lr": 3.835370957908108e-06, + "objective/entropy": -109.4366226196289, + "objective/kl": 32.577239990234375, + "objective/non_score_reward": -1.6288621425628662, + "objective/rlhf_reward": 13.332075119018555, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009311667527072132, + "policy/clipfrac_avg": 0.007746369112282991, + "policy/entropy_avg": 0.2024822235107422, + "step": 739, + "val/clipfrac_avg": 0.1542586386203766, + "val/num_eos_tokens": 140, + "val/ratio": 0.9888983964920044, + "val/ratio_var": 8.097994577838108e-05 + }, + { + "episode": 11840, + "epoch": 0.597979797979798, + "eps": 0, + "loss/policy_avg": 0.3934873938560486, + "loss/value_avg": 0.5160516500473022, + "lr": 3.822385411348429e-06, + "objective/entropy": -82.36312103271484, + "objective/kl": 38.616180419921875, + "objective/non_score_reward": -1.9308089017868042, + "objective/rlhf_reward": 12.725440979003906, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008991706417873502, + "policy/clipfrac_avg": 0.01843995228409767, + "policy/entropy_avg": 0.2245502471923828, + "step": 740, + "val/clipfrac_avg": 0.0067659346386790276, + "val/num_eos_tokens": 30, + "val/ratio": 0.988348662853241, + "val/ratio_var": 5.281575795379467e-05 + }, + { + "episode": 11856, + "epoch": 0.5987878787878788, + "eps": 0, + "loss/policy_avg": -0.06747608631849289, + "loss/value_avg": 0.3114325702190399, + "lr": 3.809408268827009e-06, + "objective/entropy": -89.22055053710938, + "objective/kl": 46.57951736450195, + "objective/non_score_reward": -2.3289759159088135, + "objective/rlhf_reward": 13.483524322509766, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000996086047962308, + "policy/clipfrac_avg": 0.005896539893001318, + "policy/entropy_avg": 0.19659423828125, + "step": 741, + "val/clipfrac_avg": 0.0699368417263031, + "val/num_eos_tokens": 107, + "val/ratio": 0.9875314235687256, + "val/ratio_var": 4.793483094545081e-05 + }, + { + "episode": 11872, + "epoch": 0.5995959595959596, + "eps": 0, + "loss/policy_avg": -0.2737995684146881, + "loss/value_avg": 0.4772384762763977, + "lr": 3.7964396229551365e-06, + "objective/entropy": -104.62603759765625, + "objective/kl": 36.626800537109375, + "objective/non_score_reward": -1.831339955329895, + "objective/rlhf_reward": 14.778034210205078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008760478813201189, + "policy/clipfrac_avg": 0.005249073728919029, + "policy/entropy_avg": 0.17531394958496094, + "step": 742, + "val/clipfrac_avg": 0.06328597664833069, + "val/num_eos_tokens": 91, + "val/ratio": 0.9885821342468262, + "val/ratio_var": 5.911157131777145e-05 + }, + { + "episode": 11888, + "epoch": 0.6004040404040404, + "eps": 0, + "loss/policy_avg": -0.11605305969715118, + "loss/value_avg": 0.30427810549736023, + "lr": 3.783479566283457e-06, + "objective/entropy": -123.75762176513672, + "objective/kl": 35.787086486816406, + "objective/non_score_reward": -1.7893544435501099, + "objective/rlhf_reward": 14.86689567565918, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007770034717395902, + "policy/clipfrac_avg": 0.0056176697835326195, + "policy/entropy_avg": 0.152313232421875, + "step": 743, + "val/clipfrac_avg": 0.048395052552223206, + "val/num_eos_tokens": 386, + "val/ratio": 0.9905653595924377, + "val/ratio_var": 5.709798642783426e-05 + }, + { + "episode": 11904, + "epoch": 0.6012121212121212, + "eps": 0, + "loss/policy_avg": -0.02294028550386429, + "loss/value_avg": 0.7253668904304504, + "lr": 3.7705281913013286e-06, + "objective/entropy": -39.3416633605957, + "objective/kl": 43.74962615966797, + "objective/non_score_reward": -2.18748140335083, + "objective/rlhf_reward": 12.648456573486328, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010895703453570604, + "policy/clipfrac_avg": 0.0052008070051670074, + "policy/entropy_avg": 0.22774124145507812, + "step": 744, + "val/clipfrac_avg": 0.01829145848751068, + "val/num_eos_tokens": 71, + "val/ratio": 0.985338568687439, + "val/ratio_var": 6.19284255662933e-05 + }, + { + "episode": 11920, + "epoch": 0.602020202020202, + "eps": 0, + "loss/policy_avg": 0.1328388750553131, + "loss/value_avg": 1.3390746116638184, + "lr": 3.7575855904361447e-06, + "objective/entropy": -79.38550567626953, + "objective/kl": 33.780616760253906, + "objective/non_score_reward": -1.6890308856964111, + "objective/rlhf_reward": 14.178156852722168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008661438478156924, + "policy/clipfrac_avg": 0.003558866214007139, + "policy/entropy_avg": 0.15964603424072266, + "step": 745, + "val/clipfrac_avg": 0.05471283197402954, + "val/num_eos_tokens": 81, + "val/ratio": 0.9897934198379517, + "val/ratio_var": 8.215933485189453e-05 + }, + { + "episode": 11936, + "epoch": 0.6028282828282828, + "eps": 0, + "loss/policy_avg": -0.29737263917922974, + "loss/value_avg": 0.6314270496368408, + "lr": 3.7446518560526903e-06, + "objective/entropy": -124.25365447998047, + "objective/kl": 42.908660888671875, + "objective/non_score_reward": -2.145432949066162, + "objective/rlhf_reward": 11.87019157409668, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009470224613323808, + "policy/clipfrac_avg": 0.006878838874399662, + "policy/entropy_avg": 0.1818103790283203, + "step": 746, + "val/clipfrac_avg": 0.0798804834485054, + "val/num_eos_tokens": 329, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 8.261724724434316e-05 + }, + { + "episode": 11952, + "epoch": 0.6036363636363636, + "eps": 0, + "loss/policy_avg": 0.12454540282487869, + "loss/value_avg": 0.4487236738204956, + "lr": 3.731727080452464e-06, + "objective/entropy": -72.31340026855469, + "objective/kl": 43.510276794433594, + "objective/non_score_reward": -2.175513982772827, + "objective/rlhf_reward": 12.347923278808594, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009941391181200743, + "policy/clipfrac_avg": 0.00658874586224556, + "policy/entropy_avg": 0.16569900512695312, + "step": 747, + "val/clipfrac_avg": 0.1540203094482422, + "val/num_eos_tokens": 77, + "val/ratio": 0.9888790845870972, + "val/ratio_var": 2.4041321012191474e-05 + }, + { + "episode": 11968, + "epoch": 0.6044444444444445, + "eps": 0, + "loss/policy_avg": -0.1547861397266388, + "loss/value_avg": 0.49022629857063293, + "lr": 3.7188113558730403e-06, + "objective/entropy": -103.35200500488281, + "objective/kl": 35.28123474121094, + "objective/non_score_reward": -1.764061689376831, + "objective/rlhf_reward": 13.27500057220459, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008191847009584308, + "policy/clipfrac_avg": 0.006343110464513302, + "policy/entropy_avg": 0.14904117584228516, + "step": 748, + "val/clipfrac_avg": 0.04430587589740753, + "val/num_eos_tokens": 124, + "val/ratio": 0.9908789396286011, + "val/ratio_var": 4.545380579656921e-05 + }, + { + "episode": 11984, + "epoch": 0.6052525252525253, + "eps": 0, + "loss/policy_avg": 0.11601711809635162, + "loss/value_avg": 0.6175556182861328, + "lr": 3.705904774487396e-06, + "objective/entropy": -111.66493225097656, + "objective/kl": 36.76349639892578, + "objective/non_score_reward": -1.838174819946289, + "objective/rlhf_reward": 13.693075180053711, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008918631356209517, + "policy/clipfrac_avg": 0.010910662822425365, + "policy/entropy_avg": 0.16454505920410156, + "step": 749, + "val/clipfrac_avg": 0.06755826622247696, + "val/num_eos_tokens": 358, + "val/ratio": 0.9886500835418701, + "val/ratio_var": 6.255816697375849e-05 + }, + { + "episode": 12000, + "epoch": 0.6060606060606061, + "eps": 0, + "loss/policy_avg": -0.04627024754881859, + "loss/value_avg": 0.6224689483642578, + "lr": 3.6930074284032613e-06, + "objective/entropy": -63.2913703918457, + "objective/kl": 32.27269744873047, + "objective/non_score_reward": -1.6136349439620972, + "objective/rlhf_reward": 14.644177436828613, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009490256779827178, + "policy/clipfrac_avg": 0.0068245502188801765, + "policy/entropy_avg": 0.17229938507080078, + "step": 750, + "val/clipfrac_avg": 0.18630267679691315, + "val/num_eos_tokens": 11, + "val/ratio": 0.9878020882606506, + "val/ratio_var": 5.455148493638262e-05 + }, + { + "episode": 12016, + "epoch": 0.6068686868686869, + "eps": 0, + "loss/policy_avg": -0.17890788614749908, + "loss/value_avg": 0.7140947580337524, + "lr": 3.6801194096624515e-06, + "objective/entropy": -54.77946472167969, + "objective/kl": 34.44614028930664, + "objective/non_score_reward": -1.7223069667816162, + "objective/rlhf_reward": 14.894880294799805, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011779924388974905, + "policy/clipfrac_avg": 0.024402525275945663, + "policy/entropy_avg": 0.2098255157470703, + "step": 751, + "val/clipfrac_avg": 0.04182715713977814, + "val/num_eos_tokens": 42, + "val/ratio": 0.9854939579963684, + "val/ratio_var": 0.00014133901277091354 + }, + { + "episode": 12032, + "epoch": 0.6076767676767677, + "eps": 0, + "loss/policy_avg": -0.0015007667243480682, + "loss/value_avg": 0.2544102668762207, + "lr": 3.6672408102402247e-06, + "objective/entropy": -70.776611328125, + "objective/kl": 37.67994689941406, + "objective/non_score_reward": -1.8839974403381348, + "objective/rlhf_reward": 12.108190536499023, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009373503271490335, + "policy/clipfrac_avg": 0.005210112780332565, + "policy/entropy_avg": 0.18710041046142578, + "step": 752, + "val/clipfrac_avg": 0.021128622815012932, + "val/num_eos_tokens": 218, + "val/ratio": 0.9874061942100525, + "val/ratio_var": 5.212551332078874e-05 + }, + { + "episode": 12048, + "epoch": 0.6084848484848485, + "eps": 0, + "loss/policy_avg": 0.05351484939455986, + "loss/value_avg": 0.28849858045578003, + "lr": 3.654371722044616e-06, + "objective/entropy": -89.17417907714844, + "objective/kl": 44.5589714050293, + "objective/non_score_reward": -2.2279486656188965, + "objective/rlhf_reward": 12.178300857543945, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009375615045428276, + "policy/clipfrac_avg": 0.005882620811462402, + "policy/entropy_avg": 0.18878555297851562, + "step": 753, + "val/clipfrac_avg": 0.021566759794950485, + "val/num_eos_tokens": 120, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 5.727734605898149e-05 + }, + { + "episode": 12064, + "epoch": 0.6092929292929293, + "eps": 0, + "loss/policy_avg": -0.12424173951148987, + "loss/value_avg": 0.359388530254364, + "lr": 3.641512236915783e-06, + "objective/entropy": -114.3979263305664, + "objective/kl": 43.28697967529297, + "objective/non_score_reward": -2.16434907913208, + "objective/rlhf_reward": 14.226276397705078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000866133370436728, + "policy/clipfrac_avg": 0.0050315093249082565, + "policy/entropy_avg": 0.16837501525878906, + "step": 754, + "val/clipfrac_avg": 0.1426960825920105, + "val/num_eos_tokens": 391, + "val/ratio": 0.9894628524780273, + "val/ratio_var": 5.016153590986505e-05 + }, + { + "episode": 12080, + "epoch": 0.6101010101010101, + "eps": 0, + "loss/policy_avg": -0.2821425795555115, + "loss/value_avg": 0.38819342851638794, + "lr": 3.6286624466253496e-06, + "objective/entropy": -164.57290649414062, + "objective/kl": 38.83340072631836, + "objective/non_score_reward": -1.9416700601577759, + "objective/rlhf_reward": 14.245829582214355, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008140793070197105, + "policy/clipfrac_avg": 0.00435154652222991, + "policy/entropy_avg": 0.15729975700378418, + "step": 755, + "val/clipfrac_avg": 0.09525477886199951, + "val/num_eos_tokens": 336, + "val/ratio": 0.989446222782135, + "val/ratio_var": 5.3185020078672096e-05 + }, + { + "episode": 12096, + "epoch": 0.610909090909091, + "eps": 0, + "loss/policy_avg": 0.05094696208834648, + "loss/value_avg": 0.4260476529598236, + "lr": 3.6158224428757538e-06, + "objective/entropy": -61.176116943359375, + "objective/kl": 32.80990219116211, + "objective/non_score_reward": -1.6404950618743896, + "objective/rlhf_reward": 16.24231719970703, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009597806492820382, + "policy/clipfrac_avg": 0.0061972904950380325, + "policy/entropy_avg": 0.16566753387451172, + "step": 756, + "val/clipfrac_avg": 0.01815173029899597, + "val/num_eos_tokens": 163, + "val/ratio": 0.9894044399261475, + "val/ratio_var": 4.711053770733997e-05 + }, + { + "episode": 12112, + "epoch": 0.6117171717171718, + "eps": 0, + "loss/policy_avg": -0.040469832718372345, + "loss/value_avg": 0.3746616244316101, + "lr": 3.602992317299594e-06, + "objective/entropy": -52.30070877075195, + "objective/kl": 34.218990325927734, + "objective/non_score_reward": -1.7109496593475342, + "objective/rlhf_reward": 13.273425102233887, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009450222714804113, + "policy/clipfrac_avg": 0.0049097794108092785, + "policy/entropy_avg": 0.1805877685546875, + "step": 757, + "val/clipfrac_avg": 0.09231511503458023, + "val/num_eos_tokens": 105, + "val/ratio": 0.9877369403839111, + "val/ratio_var": 4.407649248605594e-05 + }, + { + "episode": 12128, + "epoch": 0.6125252525252525, + "eps": 0, + "loss/policy_avg": -0.0790981650352478, + "loss/value_avg": 0.289002388715744, + "lr": 3.5901721614589725e-06, + "objective/entropy": -33.08611297607422, + "objective/kl": 42.89472961425781, + "objective/non_score_reward": -2.1447365283966064, + "objective/rlhf_reward": 14.792763710021973, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011628612410277128, + "policy/clipfrac_avg": 0.00601824838668108, + "policy/entropy_avg": 0.21850204467773438, + "step": 758, + "val/clipfrac_avg": 0.05997534096240997, + "val/num_eos_tokens": 208, + "val/ratio": 0.9858998656272888, + "val/ratio_var": 4.6025765186641365e-05 + }, + { + "episode": 12144, + "epoch": 0.6133333333333333, + "eps": 0, + "loss/policy_avg": 0.08223091810941696, + "loss/value_avg": 0.4521709978580475, + "lr": 3.5773620668448384e-06, + "objective/entropy": -57.002418518066406, + "objective/kl": 34.86107635498047, + "objective/non_score_reward": -1.743053674697876, + "objective/rlhf_reward": 12.421009063720703, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009696290362626314, + "policy/clipfrac_avg": 0.005794525146484375, + "policy/entropy_avg": 0.1826324462890625, + "step": 759, + "val/clipfrac_avg": 0.03934604302048683, + "val/num_eos_tokens": 137, + "val/ratio": 0.9880379438400269, + "val/ratio_var": 4.820780668524094e-05 + }, + { + "episode": 12160, + "epoch": 0.6141414141414141, + "eps": 0, + "loss/policy_avg": 0.012903919443488121, + "loss/value_avg": 0.22128555178642273, + "lr": 3.564562124876344e-06, + "objective/entropy": -58.77183532714844, + "objective/kl": 40.15751647949219, + "objective/non_score_reward": -2.007875919342041, + "objective/rlhf_reward": 13.171812057495117, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009755170904099941, + "policy/clipfrac_avg": 0.0075699929147958755, + "policy/entropy_avg": 0.1793661117553711, + "step": 760, + "val/clipfrac_avg": 0.007695037871599197, + "val/num_eos_tokens": 194, + "val/ratio": 0.9879107475280762, + "val/ratio_var": 9.122795017901808e-05 + }, + { + "episode": 12176, + "epoch": 0.6149494949494949, + "eps": 0, + "loss/policy_avg": 0.23269692063331604, + "loss/value_avg": 0.6034168004989624, + "lr": 3.5517724269001853e-06, + "objective/entropy": -92.11294555664062, + "objective/kl": 36.40907287597656, + "objective/non_score_reward": -1.8204538822174072, + "objective/rlhf_reward": 13.468608856201172, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009949133964255452, + "policy/clipfrac_avg": 0.012031201273202896, + "policy/entropy_avg": 0.19497203826904297, + "step": 761, + "val/clipfrac_avg": 0.03142271935939789, + "val/num_eos_tokens": 210, + "val/ratio": 0.9871051907539368, + "val/ratio_var": 7.474326412193477e-05 + }, + { + "episode": 12192, + "epoch": 0.6157575757575757, + "eps": 0, + "loss/policy_avg": -0.014487015083432198, + "loss/value_avg": 1.0809375047683716, + "lr": 3.538993064189954e-06, + "objective/entropy": -42.83490753173828, + "objective/kl": 32.94031524658203, + "objective/non_score_reward": -1.6470158100128174, + "objective/rlhf_reward": 13.567828178405762, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0011591427028179169, + "policy/clipfrac_avg": 0.004484353121370077, + "policy/entropy_avg": 0.21469688415527344, + "step": 762, + "val/clipfrac_avg": 0.1017560362815857, + "val/num_eos_tokens": 453, + "val/ratio": 0.9864501953125, + "val/ratio_var": 8.947903552325442e-05 + }, + { + "episode": 12208, + "epoch": 0.6165656565656565, + "eps": 0, + "loss/policy_avg": -0.0410991832613945, + "loss/value_avg": 1.3014378547668457, + "lr": 3.526224127945479e-06, + "objective/entropy": -71.51974487304688, + "objective/kl": 31.28769302368164, + "objective/non_score_reward": -1.5643846988677979, + "objective/rlhf_reward": 15.216865539550781, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007811100222170353, + "policy/clipfrac_avg": 0.0053521376103162766, + "policy/entropy_avg": 0.16764509677886963, + "step": 763, + "val/clipfrac_avg": 0.059704042971134186, + "val/num_eos_tokens": 203, + "val/ratio": 0.9894566535949707, + "val/ratio_var": 6.841208232799545e-05 + }, + { + "episode": 12224, + "epoch": 0.6173737373737374, + "eps": 0, + "loss/policy_avg": 0.112126424908638, + "loss/value_avg": 0.725554347038269, + "lr": 3.5134657092921877e-06, + "objective/entropy": -99.9854736328125, + "objective/kl": 41.09782028198242, + "objective/non_score_reward": -2.0548908710479736, + "objective/rlhf_reward": 10.624795913696289, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009035035036504269, + "policy/clipfrac_avg": 0.010427422821521759, + "policy/entropy_avg": 0.17998504638671875, + "step": 764, + "val/clipfrac_avg": 0.12764863669872284, + "val/num_eos_tokens": 226, + "val/ratio": 0.988477349281311, + "val/ratio_var": 8.909796451916918e-05 + }, + { + "episode": 12240, + "epoch": 0.6181818181818182, + "eps": 0, + "loss/policy_avg": 0.37758830189704895, + "loss/value_avg": 0.7465800046920776, + "lr": 3.500717899280442e-06, + "objective/entropy": -135.23390197753906, + "objective/kl": 41.466583251953125, + "objective/non_score_reward": -2.073329210281372, + "objective/rlhf_reward": 12.94229507446289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009621024364605546, + "policy/clipfrac_avg": 0.00628144945949316, + "policy/entropy_avg": 0.17942047119140625, + "step": 765, + "val/clipfrac_avg": 0.10096652805805206, + "val/num_eos_tokens": 233, + "val/ratio": 0.9877568483352661, + "val/ratio_var": 7.067532715154812e-05 + }, + { + "episode": 12256, + "epoch": 0.618989898989899, + "eps": 0, + "loss/policy_avg": -0.10782650113105774, + "loss/value_avg": 0.4996359944343567, + "lr": 3.4879807888849026e-06, + "objective/entropy": -86.19502258300781, + "objective/kl": 39.61420440673828, + "objective/non_score_reward": -1.9807100296020508, + "objective/rlhf_reward": 13.706789016723633, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009516358841210604, + "policy/clipfrac_avg": 0.01008635200560093, + "policy/entropy_avg": 0.17633438110351562, + "step": 766, + "val/clipfrac_avg": 0.05575938522815704, + "val/num_eos_tokens": 426, + "val/ratio": 0.9884860515594482, + "val/ratio_var": 5.413329563452862e-05 + }, + { + "episode": 12272, + "epoch": 0.6197979797979798, + "eps": 0, + "loss/policy_avg": 0.0611935630440712, + "loss/value_avg": 0.40072089433670044, + "lr": 3.475254469003865e-06, + "objective/entropy": -78.26728057861328, + "objective/kl": 38.48719787597656, + "objective/non_score_reward": -1.9243600368499756, + "objective/rlhf_reward": 11.661577224731445, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009466722258366644, + "policy/clipfrac_avg": 0.0042708818800747395, + "policy/entropy_avg": 0.17498779296875, + "step": 767, + "val/clipfrac_avg": 0.1151452288031578, + "val/num_eos_tokens": 415, + "val/ratio": 0.9874659776687622, + "val/ratio_var": 7.443072536261752e-05 + }, + { + "episode": 12288, + "epoch": 0.6206060606060606, + "eps": 0, + "loss/policy_avg": -0.07818017899990082, + "loss/value_avg": 0.588293731212616, + "lr": 3.4625390304586224e-06, + "objective/entropy": -45.31693649291992, + "objective/kl": 41.06260681152344, + "objective/non_score_reward": -2.0531303882598877, + "objective/rlhf_reward": 13.860933303833008, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0011651598615571856, + "policy/clipfrac_avg": 0.0047182347625494, + "policy/entropy_avg": 0.2090606689453125, + "step": 768, + "val/clipfrac_avg": 0.16338574886322021, + "val/num_eos_tokens": 9, + "val/ratio": 0.986497700214386, + "val/ratio_var": 8.175932453013957e-05 + }, + { + "episode": 12304, + "epoch": 0.6214141414141414, + "eps": 0, + "loss/policy_avg": -0.3265213966369629, + "loss/value_avg": 0.5878744125366211, + "lr": 3.4498345639928167e-06, + "objective/entropy": -103.96510314941406, + "objective/kl": 35.794551849365234, + "objective/non_score_reward": -1.7897276878356934, + "objective/rlhf_reward": 13.507146835327148, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010084104724228382, + "policy/clipfrac_avg": 0.0070042358711361885, + "policy/entropy_avg": 0.19075870513916016, + "step": 769, + "val/clipfrac_avg": 0.12878292798995972, + "val/num_eos_tokens": 8, + "val/ratio": 0.9861502051353455, + "val/ratio_var": 8.973718649940565e-05 + }, + { + "episode": 12320, + "epoch": 0.6222222222222222, + "eps": 0, + "loss/policy_avg": -0.24864813685417175, + "loss/value_avg": 0.703667402267456, + "lr": 3.4371411602717785e-06, + "objective/entropy": -92.59934997558594, + "objective/kl": 31.87890625, + "objective/non_score_reward": -1.5939452648162842, + "objective/rlhf_reward": 14.429492950439453, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007836055592633784, + "policy/clipfrac_avg": 0.004061062820255756, + "policy/entropy_avg": 0.13613224029541016, + "step": 770, + "val/clipfrac_avg": 0.14356926083564758, + "val/num_eos_tokens": 22, + "val/ratio": 0.9909623265266418, + "val/ratio_var": 6.252773891901597e-05 + }, + { + "episode": 12336, + "epoch": 0.623030303030303, + "eps": 0, + "loss/policy_avg": -0.11106158792972565, + "loss/value_avg": 0.7719787359237671, + "lr": 3.424458909881897e-06, + "objective/entropy": -201.825927734375, + "objective/kl": 32.05082321166992, + "objective/non_score_reward": -1.602541208267212, + "objective/rlhf_reward": 13.842771530151367, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009224842651747167, + "policy/clipfrac_avg": 0.004090993665158749, + "policy/entropy_avg": 0.19452500343322754, + "step": 771, + "val/clipfrac_avg": 0.06381858140230179, + "val/num_eos_tokens": 137, + "val/ratio": 0.9873221516609192, + "val/ratio_var": 0.00010800129530252889 + }, + { + "episode": 12352, + "epoch": 0.6238383838383839, + "eps": 0, + "loss/policy_avg": -0.3147702217102051, + "loss/value_avg": 1.0198739767074585, + "lr": 3.411787903329963e-06, + "objective/entropy": -97.80130767822266, + "objective/kl": 28.94087791442871, + "objective/non_score_reward": -1.4470438957214355, + "objective/rlhf_reward": 13.572486877441406, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008233180269598961, + "policy/clipfrac_avg": 0.007523335050791502, + "policy/entropy_avg": 0.17452716827392578, + "step": 772, + "val/clipfrac_avg": 0.021688329055905342, + "val/num_eos_tokens": 116, + "val/ratio": 0.9885031580924988, + "val/ratio_var": 6.793635111534968e-05 + }, + { + "episode": 12368, + "epoch": 0.6246464646464647, + "eps": 0, + "loss/policy_avg": -0.019919998943805695, + "loss/value_avg": 0.31691867113113403, + "lr": 3.3991282310425276e-06, + "objective/entropy": -61.794166564941406, + "objective/kl": 33.001129150390625, + "objective/non_score_reward": -1.6500564813613892, + "objective/rlhf_reward": 12.943693161010742, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009417567634955049, + "policy/clipfrac_avg": 0.005282199941575527, + "policy/entropy_avg": 0.1732311248779297, + "step": 773, + "val/clipfrac_avg": 0.05137361213564873, + "val/num_eos_tokens": 13, + "val/ratio": 0.9889352321624756, + "val/ratio_var": 3.9570189983351156e-05 + }, + { + "episode": 12384, + "epoch": 0.6254545454545455, + "eps": 0, + "loss/policy_avg": 0.010051839053630829, + "loss/value_avg": 0.5414019227027893, + "lr": 3.386479983365251e-06, + "objective/entropy": -73.91504669189453, + "objective/kl": 29.43255615234375, + "objective/non_score_reward": -1.4716278314590454, + "objective/rlhf_reward": 13.293996810913086, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0010180522222071886, + "policy/clipfrac_avg": 0.0050889430567622185, + "policy/entropy_avg": 0.2153911590576172, + "step": 774, + "val/clipfrac_avg": 0.041913747787475586, + "val/num_eos_tokens": 214, + "val/ratio": 0.986315906047821, + "val/ratio_var": 8.102346328087151e-05 + }, + { + "episode": 12400, + "epoch": 0.6262626262626263, + "eps": 0, + "loss/policy_avg": -0.040474407374858856, + "loss/value_avg": 0.30427631735801697, + "lr": 3.3738432505622653e-06, + "objective/entropy": -75.78406524658203, + "objective/kl": 31.846160888671875, + "objective/non_score_reward": -1.5923081636428833, + "objective/rlhf_reward": 11.946754455566406, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009044273756444454, + "policy/clipfrac_avg": 0.005002772435545921, + "policy/entropy_avg": 0.18775272369384766, + "step": 775, + "val/clipfrac_avg": 0.04153668135404587, + "val/num_eos_tokens": 243, + "val/ratio": 0.9877529144287109, + "val/ratio_var": 5.462392800836824e-05 + }, + { + "episode": 12416, + "epoch": 0.6270707070707071, + "eps": 0, + "loss/policy_avg": -0.25257226824760437, + "loss/value_avg": 0.47636741399765015, + "lr": 3.3612181228155237e-06, + "objective/entropy": -97.84140014648438, + "objective/kl": 35.974266052246094, + "objective/non_score_reward": -1.798713207244873, + "objective/rlhf_reward": 13.685661315917969, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009363207500427961, + "policy/clipfrac_avg": 0.0035052683670073748, + "policy/entropy_avg": 0.18935298919677734, + "step": 776, + "val/clipfrac_avg": 0.07260937243700027, + "val/num_eos_tokens": 26, + "val/ratio": 0.9877586364746094, + "val/ratio_var": 0.00010292992374161258 + }, + { + "episode": 12432, + "epoch": 0.6278787878787879, + "eps": 0, + "loss/policy_avg": 0.41764986515045166, + "loss/value_avg": 0.8629061579704285, + "lr": 3.3486046902241663e-06, + "objective/entropy": -40.24072265625, + "objective/kl": 29.197486877441406, + "objective/non_score_reward": -1.4598743915557861, + "objective/rlhf_reward": 12.883874893188477, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010831760009750724, + "policy/clipfrac_avg": 0.02700810506939888, + "policy/entropy_avg": 0.16212844848632812, + "step": 777, + "val/clipfrac_avg": 0.08423963189125061, + "val/num_eos_tokens": 192, + "val/ratio": 0.9885735511779785, + "val/ratio_var": 8.724266081117094e-05 + }, + { + "episode": 12448, + "epoch": 0.6286868686868687, + "eps": 0, + "loss/policy_avg": 0.2242026925086975, + "loss/value_avg": 0.9421164989471436, + "lr": 3.336003042803861e-06, + "objective/entropy": -69.478271484375, + "objective/kl": 33.57006072998047, + "objective/non_score_reward": -1.6785030364990234, + "objective/rlhf_reward": 14.055871963500977, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008899248205125332, + "policy/clipfrac_avg": 0.006825867109000683, + "policy/entropy_avg": 0.14767837524414062, + "step": 778, + "val/clipfrac_avg": 0.1501787304878235, + "val/num_eos_tokens": 109, + "val/ratio": 0.9905418753623962, + "val/ratio_var": 4.142183388466947e-05 + }, + { + "episode": 12464, + "epoch": 0.6294949494949494, + "eps": 0, + "loss/policy_avg": 0.03614666312932968, + "loss/value_avg": 0.7387021780014038, + "lr": 3.3234132704861786e-06, + "objective/entropy": -182.4083251953125, + "objective/kl": 31.944690704345703, + "objective/non_score_reward": -1.5972344875335693, + "objective/rlhf_reward": 12.043390274047852, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.000871892087161541, + "policy/clipfrac_avg": 0.004613575059920549, + "policy/entropy_avg": 0.1733226776123047, + "step": 779, + "val/clipfrac_avg": 0.1070820540189743, + "val/num_eos_tokens": 28, + "val/ratio": 0.9880519509315491, + "val/ratio_var": 9.137512097368017e-05 + }, + { + "episode": 12480, + "epoch": 0.6303030303030303, + "eps": 0, + "loss/policy_avg": 0.491243839263916, + "loss/value_avg": 1.7449349164962769, + "lr": 3.310835463117942e-06, + "objective/entropy": -80.43102264404297, + "objective/kl": 22.97904396057129, + "objective/non_score_reward": -1.1489522457122803, + "objective/rlhf_reward": 13.28073501586914, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000753985601477325, + "policy/clipfrac_avg": 0.013304230757057667, + "policy/entropy_avg": 0.16235482692718506, + "step": 780, + "val/clipfrac_avg": 0.12828537821769714, + "val/num_eos_tokens": 261, + "val/ratio": 0.991666316986084, + "val/ratio_var": 4.57862151961308e-05 + }, + { + "episode": 12496, + "epoch": 0.6311111111111111, + "eps": 0, + "loss/policy_avg": -0.02106083557009697, + "loss/value_avg": 0.26465463638305664, + "lr": 3.2982697104605855e-06, + "objective/entropy": -96.58609008789062, + "objective/kl": 37.96983337402344, + "objective/non_score_reward": -1.898491621017456, + "objective/rlhf_reward": 12.742133140563965, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008919077808968723, + "policy/clipfrac_avg": 0.004138793330639601, + "policy/entropy_avg": 0.176055908203125, + "step": 781, + "val/clipfrac_avg": 0.022776879370212555, + "val/num_eos_tokens": 232, + "val/ratio": 0.9880313873291016, + "val/ratio_var": 5.30094766872935e-05 + }, + { + "episode": 12512, + "epoch": 0.6319191919191919, + "eps": 0, + "loss/policy_avg": -0.21595685184001923, + "loss/value_avg": 0.1668311059474945, + "lr": 3.2857161021895125e-06, + "objective/entropy": -113.97618103027344, + "objective/kl": 30.898021697998047, + "objective/non_score_reward": -1.5449011325836182, + "objective/rlhf_reward": 15.892599105834961, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0008122409926727414, + "policy/clipfrac_avg": 0.007180417887866497, + "policy/entropy_avg": 0.15228557586669922, + "step": 782, + "val/clipfrac_avg": 0.04710119590163231, + "val/num_eos_tokens": 88, + "val/ratio": 0.9905120134353638, + "val/ratio_var": 4.6862616727594286e-05 + }, + { + "episode": 12528, + "epoch": 0.6327272727272727, + "eps": 0, + "loss/policy_avg": -0.02830284647643566, + "loss/value_avg": 0.470698744058609, + "lr": 3.273174727893463e-06, + "objective/entropy": -119.84375762939453, + "objective/kl": 35.46149444580078, + "objective/non_score_reward": -1.773074746131897, + "objective/rlhf_reward": 13.00036334991455, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000846280949190259, + "policy/clipfrac_avg": 0.006307992152869701, + "policy/entropy_avg": 0.15985870361328125, + "step": 783, + "val/clipfrac_avg": 0.09303553402423859, + "val/num_eos_tokens": 242, + "val/ratio": 0.9886760711669922, + "val/ratio_var": 8.341170905623585e-05 + }, + { + "episode": 12544, + "epoch": 0.6335353535353535, + "eps": 0, + "loss/policy_avg": -0.03483296558260918, + "loss/value_avg": 0.2278536707162857, + "lr": 3.2606456770738636e-06, + "objective/entropy": -32.83402633666992, + "objective/kl": 35.053001403808594, + "objective/non_score_reward": -1.7526500225067139, + "objective/rlhf_reward": 13.708287239074707, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010523913661018014, + "policy/clipfrac_avg": 0.005916148889809847, + "policy/entropy_avg": 0.2171039581298828, + "step": 784, + "val/clipfrac_avg": 0.007893375121057034, + "val/num_eos_tokens": 463, + "val/ratio": 0.9859672784805298, + "val/ratio_var": 5.139111090102233e-05 + }, + { + "episode": 12560, + "epoch": 0.6343434343434343, + "eps": 0, + "loss/policy_avg": 0.17171788215637207, + "loss/value_avg": 0.5937256217002869, + "lr": 3.2481290391442034e-06, + "objective/entropy": -109.50154113769531, + "objective/kl": 41.38114929199219, + "objective/non_score_reward": -2.0690574645996094, + "objective/rlhf_reward": 13.82156753540039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009320294484496117, + "policy/clipfrac_avg": 0.005337849725037813, + "policy/entropy_avg": 0.1828467845916748, + "step": 785, + "val/clipfrac_avg": 0.0434221550822258, + "val/num_eos_tokens": 252, + "val/ratio": 0.9880971908569336, + "val/ratio_var": 7.803420157870278e-05 + }, + { + "episode": 12576, + "epoch": 0.6351515151515151, + "eps": 0, + "loss/policy_avg": -0.12326152622699738, + "loss/value_avg": 0.22453081607818604, + "lr": 3.235624903429374e-06, + "objective/entropy": -80.62982940673828, + "objective/kl": 39.33965301513672, + "objective/non_score_reward": -1.9669828414916992, + "objective/rlhf_reward": 14.3767671585083, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009726607240736485, + "policy/clipfrac_avg": 0.005861825309693813, + "policy/entropy_avg": 0.18674850463867188, + "step": 786, + "val/clipfrac_avg": 0.04197417199611664, + "val/num_eos_tokens": 55, + "val/ratio": 0.9867269396781921, + "val/ratio_var": 3.261497477069497e-05 + }, + { + "episode": 12592, + "epoch": 0.635959595959596, + "eps": 0, + "loss/policy_avg": 0.02891068160533905, + "loss/value_avg": 0.2414318025112152, + "lr": 3.2231333591650567e-06, + "objective/entropy": -64.31414031982422, + "objective/kl": 46.58558654785156, + "objective/non_score_reward": -2.3292794227600098, + "objective/rlhf_reward": 14.576970100402832, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009546708315610886, + "policy/clipfrac_avg": 0.00552853848785162, + "policy/entropy_avg": 0.189300537109375, + "step": 787, + "val/clipfrac_avg": 0.0013757861452177167, + "val/num_eos_tokens": 363, + "val/ratio": 0.987205982208252, + "val/ratio_var": 3.5099023079965264e-05 + }, + { + "episode": 12608, + "epoch": 0.6367676767676768, + "eps": 0, + "loss/policy_avg": -0.19538097083568573, + "loss/value_avg": 0.7160595655441284, + "lr": 3.210654495497068e-06, + "objective/entropy": -57.81846237182617, + "objective/kl": 42.85956573486328, + "objective/non_score_reward": -2.1429784297943115, + "objective/rlhf_reward": 12.14999008178711, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0011135776294395328, + "policy/clipfrac_avg": 0.009378955699503422, + "policy/entropy_avg": 0.2034454345703125, + "step": 788, + "val/clipfrac_avg": 0.1690627932548523, + "val/num_eos_tokens": 481, + "val/ratio": 0.9861621856689453, + "val/ratio_var": 9.24188134376891e-05 + }, + { + "episode": 12624, + "epoch": 0.6375757575757576, + "eps": 0, + "loss/policy_avg": 0.048018310219049454, + "loss/value_avg": 0.4312187135219574, + "lr": 3.198188401480734e-06, + "objective/entropy": -76.67124938964844, + "objective/kl": 36.849369049072266, + "objective/non_score_reward": -1.8424686193466187, + "objective/rlhf_reward": 12.25128173828125, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009952555410563946, + "policy/clipfrac_avg": 0.004150367341935635, + "policy/entropy_avg": 0.18071389198303223, + "step": 789, + "val/clipfrac_avg": 0.08804323524236679, + "val/num_eos_tokens": 357, + "val/ratio": 0.9867403507232666, + "val/ratio_var": 0.00013288675108924508 + }, + { + "episode": 12640, + "epoch": 0.6383838383838384, + "eps": 0, + "loss/policy_avg": 0.18971526622772217, + "loss/value_avg": 0.6626036167144775, + "lr": 3.185735166080244e-06, + "objective/entropy": -126.28926086425781, + "objective/kl": 33.96638488769531, + "objective/non_score_reward": -1.6983189582824707, + "objective/rlhf_reward": 12.973556518554688, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0007302669691853225, + "policy/clipfrac_avg": 0.008397979661822319, + "policy/entropy_avg": 0.14786386489868164, + "step": 790, + "val/clipfrac_avg": 0.06916166841983795, + "val/num_eos_tokens": 600, + "val/ratio": 0.9910507798194885, + "val/ratio_var": 5.283964492264204e-05 + }, + { + "episode": 12656, + "epoch": 0.6391919191919192, + "eps": 0, + "loss/policy_avg": -0.4279332160949707, + "loss/value_avg": 0.5117957592010498, + "lr": 3.173294878168025e-06, + "objective/entropy": -75.64164733886719, + "objective/kl": 40.16947555541992, + "objective/non_score_reward": -2.0084738731384277, + "objective/rlhf_reward": 12.86652660369873, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008897786610759795, + "policy/clipfrac_avg": 0.004540015943348408, + "policy/entropy_avg": 0.18616324663162231, + "step": 791, + "val/clipfrac_avg": 0.0338541679084301, + "val/num_eos_tokens": 224, + "val/ratio": 0.9873502254486084, + "val/ratio_var": 6.438903801608831e-05 + }, + { + "episode": 12672, + "epoch": 0.64, + "eps": 0, + "loss/policy_avg": -0.30223172903060913, + "loss/value_avg": 0.7414761185646057, + "lr": 3.160867626524107e-06, + "objective/entropy": -36.1086540222168, + "objective/kl": 35.08636474609375, + "objective/non_score_reward": -1.7543182373046875, + "objective/rlhf_reward": 14.995681762695312, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010164354462176561, + "policy/clipfrac_avg": 0.0033426620066165924, + "policy/entropy_avg": 0.20384597778320312, + "step": 792, + "val/clipfrac_avg": 0.08375494927167892, + "val/num_eos_tokens": 121, + "val/ratio": 0.9862762093544006, + "val/ratio_var": 8.216432615881786e-05 + }, + { + "episode": 12688, + "epoch": 0.6408080808080808, + "eps": 0, + "loss/policy_avg": 0.11556120216846466, + "loss/value_avg": 0.44696569442749023, + "lr": 3.1484534998354832e-06, + "objective/entropy": -52.89218521118164, + "objective/kl": 33.01978302001953, + "objective/non_score_reward": -1.650989294052124, + "objective/rlhf_reward": 12.270885467529297, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009637182811275125, + "policy/clipfrac_avg": 0.006595691200345755, + "policy/entropy_avg": 0.18551349639892578, + "step": 793, + "val/clipfrac_avg": 0.0313958078622818, + "val/num_eos_tokens": 483, + "val/ratio": 0.987555742263794, + "val/ratio_var": 4.9779242544900626e-05 + }, + { + "episode": 12704, + "epoch": 0.6416161616161616, + "eps": 0, + "loss/policy_avg": -0.11985203623771667, + "loss/value_avg": 0.5595318078994751, + "lr": 3.136052586695481e-06, + "objective/entropy": -44.2855110168457, + "objective/kl": 38.74113845825195, + "objective/non_score_reward": -1.9370570182800293, + "objective/rlhf_reward": 13.531692504882812, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009892593370750546, + "policy/clipfrac_avg": 0.006002710200846195, + "policy/entropy_avg": 0.20067405700683594, + "step": 794, + "val/clipfrac_avg": 0.051367055624723434, + "val/num_eos_tokens": 146, + "val/ratio": 0.98707115650177, + "val/ratio_var": 6.210069841472432e-05 + }, + { + "episode": 12720, + "epoch": 0.6424242424242425, + "eps": 0, + "loss/policy_avg": 0.06161116436123848, + "loss/value_avg": 0.20387178659439087, + "lr": 3.12366497560313e-06, + "objective/entropy": -47.063812255859375, + "objective/kl": 36.60810852050781, + "objective/non_score_reward": -1.8304054737091064, + "objective/rlhf_reward": 13.661782264709473, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000998336123302579, + "policy/clipfrac_avg": 0.004183639772236347, + "policy/entropy_avg": 0.19140625, + "step": 795, + "val/clipfrac_avg": 0.08834637701511383, + "val/num_eos_tokens": 342, + "val/ratio": 0.9874628782272339, + "val/ratio_var": 3.3792719477787614e-05 + }, + { + "episode": 12736, + "epoch": 0.6432323232323233, + "eps": 0, + "loss/policy_avg": 0.02652081847190857, + "loss/value_avg": 0.4432865381240845, + "lr": 3.1112907549625304e-06, + "objective/entropy": -104.79203796386719, + "objective/kl": 35.12114715576172, + "objective/non_score_reward": -1.7560575008392334, + "objective/rlhf_reward": 13.486129760742188, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008146626059897244, + "policy/clipfrac_avg": 0.0056906514801084995, + "policy/entropy_avg": 0.1502237319946289, + "step": 796, + "val/clipfrac_avg": 0.061473242938518524, + "val/num_eos_tokens": 748, + "val/ratio": 0.9898946285247803, + "val/ratio_var": 4.619114042725414e-05 + }, + { + "episode": 12752, + "epoch": 0.6440404040404041, + "eps": 0, + "loss/policy_avg": 0.06834741681814194, + "loss/value_avg": 0.5731651782989502, + "lr": 3.098930013082223e-06, + "objective/entropy": -88.31727600097656, + "objective/kl": 38.224117279052734, + "objective/non_score_reward": -1.9112060070037842, + "objective/rlhf_reward": 14.698169708251953, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008200570591725409, + "policy/clipfrac_avg": 0.007693252060562372, + "policy/entropy_avg": 0.1607208251953125, + "step": 797, + "val/clipfrac_avg": 0.11237858235836029, + "val/num_eos_tokens": 240, + "val/ratio": 0.9905504584312439, + "val/ratio_var": 4.706854815594852e-05 + }, + { + "episode": 12768, + "epoch": 0.6448484848484849, + "eps": 0, + "loss/policy_avg": -0.09761235117912292, + "loss/value_avg": 0.7088909149169922, + "lr": 3.0865828381745515e-06, + "objective/entropy": -124.7913589477539, + "objective/kl": 29.702392578125, + "objective/non_score_reward": -1.4851195812225342, + "objective/rlhf_reward": 13.68675422668457, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008782441727817059, + "policy/clipfrac_avg": 0.00598434591665864, + "policy/entropy_avg": 0.16924285888671875, + "step": 798, + "val/clipfrac_avg": 0.030641667544841766, + "val/num_eos_tokens": 454, + "val/ratio": 0.9882635474205017, + "val/ratio_var": 7.515518518630415e-05 + }, + { + "episode": 12784, + "epoch": 0.6456565656565657, + "eps": 0, + "loss/policy_avg": 0.14146611094474792, + "loss/value_avg": 0.7321012616157532, + "lr": 3.074249318355046e-06, + "objective/entropy": -101.5077133178711, + "objective/kl": 25.74894142150879, + "objective/non_score_reward": -1.2874470949172974, + "objective/rlhf_reward": 13.735990524291992, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007512210868299007, + "policy/clipfrac_avg": 0.011982694268226624, + "policy/entropy_avg": 0.15892601013183594, + "step": 799, + "val/clipfrac_avg": 0.012677992694079876, + "val/num_eos_tokens": 12, + "val/ratio": 0.9910134077072144, + "val/ratio_var": 6.09022390563041e-05 + }, + { + "episode": 12800, + "epoch": 0.6464646464646465, + "eps": 0, + "loss/policy_avg": -0.042851828038692474, + "loss/value_avg": 0.3452759385108948, + "lr": 3.0619295416417856e-06, + "objective/entropy": -68.88214874267578, + "objective/kl": 34.023399353027344, + "objective/non_score_reward": -1.7011699676513672, + "objective/rlhf_reward": 12.494142532348633, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010239251423627138, + "policy/clipfrac_avg": 0.010894561186432838, + "policy/entropy_avg": 0.17738628387451172, + "step": 800, + "val/clipfrac_avg": 0.026722928509116173, + "val/num_eos_tokens": 67, + "val/ratio": 0.9888595342636108, + "val/ratio_var": 6.888628558954224e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-800/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-800/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/README.md b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6e659f132b100164f71e4822c51547d23fbc37f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/README.md @@ -0,0 +1,207 @@ +--- +base_model: /DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.16.0 \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/adapter_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..730a3abc191dadc253e7a37b356cef87c16c16ec --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": [], + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/adapter_model.safetensors b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac3c78a8fcc5aa3875757603bf416f9ac7dde7c3 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3480332e6dbae56126cb9c88ef59ef2c2e44f7416cf24117e8cb4d20af81fd5 +size 42002584 diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/additional_config.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/additional_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe5159d1d10a158affb4d328c70025d891e16d8 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/additional_config.json @@ -0,0 +1 @@ +{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/args.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/args.json new file mode 100644 index 0000000000000000000000000000000000000000..0255a204a288c36c336a0aa5d31fecf2f829ec70 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/args.json @@ -0,0 +1,472 @@ +{ + "output_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 2, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.05, + "warmup_steps": 0, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 5, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 100.0, + "save_total_limit": 12, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "use_ipex": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 100.0, + "dataloader_num_workers": 8, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": null, + "greater_is_better": null, + "ignore_data_skip": false, + "fsdp": "", + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 0, + "allgather_partitions": true, + "allgather_bucket_size": 200000000.0, + "overlap_comm": false, + "reduce_scatter": true, + "reduce_bucket_size": 200000000.0, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch", + "optim_args": null, + "adafactor": false, + "group_by_length": false, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "ddp_find_unused_parameters": null, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": false, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": null, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "optimizer": null, + "use_logits_to_keep": null, + "channels": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "model": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "model_type": "llama3_1", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": null, + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "llama3_2", + "system": null, + "max_length": 4096, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "left", + "loss_scale": "last_round", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/DIR-reproduction/scripts/alpaca-gpt4-data-en#20000" + ], + "val_dataset": [], + "split_dataset_ratio": 0.01, + "dataset_num_proc": 8, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": null, + "model_author": null, + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 2048, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9, + "repetition_penalty": 1.0, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "lora", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "loss_type": null, + "metric": null, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "sft_alpha": 0, + "reward_model": [ + "/DIR-reproduction/reward_models/my_outputs/DIR-Llama3.1-8B-RM" + ], + "reward_adapters": [], + "reward_model_type": [ + "llama3_1" + ], + "reward_model_revision": null, + "num_ppo_epochs": 4, + "whiten_rewards": false, + "kl_coef": 0.05, + "cliprange": 0.2, + "vf_coef": 0.1, + "cliprange_value": 0.2, + "gamma": 1.0, + "lam": 0.95, + "num_mini_batches": 1, + "local_rollout_forward_batch_size": 64, + "num_sample_generations": 10, + "response_length": 2048, + "missing_eos_penalty": null, + "vllm_gpu_memory_utilization": 0.9, + "vllm_tensor_parallel_size": 1, + "vllm_pipeline_parallel_size": 1, + "vllm_enable_expert_parallel": false, + "vllm_max_num_seqs": 256, + "vllm_max_model_len": null, + "vllm_disable_custom_all_reduce": true, + "vllm_enforce_eager": false, + "vllm_limit_mm_per_prompt": {}, + "vllm_max_lora_rank": 16, + "vllm_enable_prefix_caching": true, + "vllm_use_async_engine": false, + "vllm_quantization": null, + "vllm_data_parallel_size": 1, + "gpu_memory_utilization": null, + "tensor_parallel_size": null, + "limit_mm_per_prompt": null, + "data_parallel_size": null, + "use_async_engine": null, + "epsilon": 0.2, + "epsilon_high": null, + "delta": null, + "vllm_mode": "colocate", + "vllm_server_base_url": null, + "vllm_server_host": null, + "vllm_server_port": [ + 8000 + ], + "vllm_server_timeout": 240.0, + "cosine_min_len_value_wrong": -0.5, + "cosine_max_len_value_wrong": 0.0, + "cosine_min_len_value_correct": 1.0, + "cosine_max_len_value_correct": 0.5, + "cosine_max_len": null, + "repetition_n_grams": 3, + "repetition_max_penalty": -1.0, + "reward_model_plugin": null, + "sync_ref_model": false, + "ref_model_sync_steps": 512, + "ref_model_mixup_alpha": 0.6, + "async_generate": false, + "sleep_level": 0, + "move_model_batches": null, + "offload_optimizer": false, + "offload_model": false, + "gc_collect_after_offload": false, + "multi_turn_func": null, + "multi_turn_scheduler": null, + "max_turns": null, + "completion_length_limit_scope": "per_round", + "dynamic_sample": false, + "max_resample_times": 3, + "overlong_filter": false, + "soft_max_length": null, + "soft_cache_length": null, + "scale_rewards": true, + "log_entropy": false, + "top_entropy_quantile": 1.0, + "importance_sampling_level": "token", + "wandb_log_unique_prompts": null, + "generation_batch_size": null, + "steps_per_generation": null, + "num_generations": 8, + "reward_funcs": [], + "reward_weights": null, + "log_completions": false, + "use_vllm": false, + "num_iterations": 1, + "teacher_model": null, + "teacher_adapters": [], + "teacher_model_type": null, + "teacher_model_revision": null, + "rlhf_type": "ppo", + "ref_model": null, + "ref_model_type": null, + "ref_model_revision": null, + "beta": 0.1, + "label_smoothing": 0, + "max_completion_length": 2048, + "rpo_alpha": 1.0, + "loss_weights": null, + "cpo_alpha": 1.0, + "simpo_gamma": 1, + "desirable_weight": 1.0, + "undesirable_weight": 1.0, + "center_rewards_coefficient": null, + "lmbda": 0.5, + "seq_kd": false, + "rank": 0, + "global_world_size": 8, + "local_world_size": 8, + "model_suffix": "Meta-Llama-3.1-8B-Instruct", + "model_info": "ModelInfo(model_type='llama3_1', model_dir='/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='llama3_1', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B', hf_model_id='meta-llama/Meta-Llama-3.1-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B', hf_model_id='meta-llama/Meta-Llama-3.1-70B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B', hf_model_id='meta-llama/Meta-Llama-3.1-405B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-70B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8', hf_model_id='meta-llama/Meta-Llama-3.1-405B-Instruct-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit', hf_model_id='unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', hf_model_id='hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF', hf_model_id='nvidia/Llama-3.1-Nemotron-70B-Instruct-HF', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='llama3_2', get_function=, model_arch='llama', architectures=['LlamaForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.43'], tags=[])", + "model_dir": "/DIR-reproduction/scripts/Meta-Llama-3.1-8B-Instruct", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "PPOConfig(output_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/runs', logging_strategy=, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=, save_steps=100, save_total_limit=12, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 0, 'allgather_partitions': True, 'allgather_bucket_size': 200000000.0, 'overlap_comm': False, 'reduce_scatter': True, 'reduce_bucket_size': 200000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, dataset_num_proc=8, num_mini_batches=1, total_episodes=None, local_rollout_forward_batch_size=64, num_sample_generations=10, response_length=2048, stop_token=None, stop_token_id=None, temperature=0.7, missing_eos_penalty=None, sft_model_path='EleutherAI/pythia-160m', world_size=8, num_total_batches=None, micro_batch_size=None, local_batch_size=None, batch_size=None, local_mini_batch_size=None, mini_batch_size=None, exp_name='ppo_config', reward_model_path='EleutherAI/pythia-160m', model_adapter_name=None, ref_adapter_name=None, num_ppo_epochs=4, whiten_rewards=False, kl_coef=0.05, kl_estimator='k1', cliprange=0.2, vf_coef=0.1, cliprange_value=0.2, gamma=1.0, lam=0.95, ds3_gather_for_generation=True, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, use_logits_to_keep=None, channels=None, resume_only_model=False, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, sft_alpha=0, train_type='lora', local_repo_path=None, galore_config=None)" +} \ No newline at end of file diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/trainer_state.json b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e69221180901ce4c79f95b91ca6db7405bd801fe --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/trainer_state.json @@ -0,0 +1,18935 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "episode": 14400, + "epoch": 0.7272727272727273, + "eval_steps": 100, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "episode": 16, + "epoch": 0.0008080808080808081, + "eps": 0, + "loss/policy_avg": -0.2878437042236328, + "loss/value_avg": 10.6063871383667, + "lr": 0.0, + "objective/entropy": -74.55732727050781, + "objective/kl": 4.434216499328613, + "objective/non_score_reward": -0.22171086072921753, + "objective/rlhf_reward": 9.188445091247559, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0009155935840681195, + "policy/clipfrac_avg": 0.0011241745669394732, + "policy/entropy_avg": 0.19521331787109375, + "step": 1, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 545, + "val/ratio": 0.9892521500587463, + "val/ratio_var": 5.706197043764405e-05 + }, + { + "episode": 32, + "epoch": 0.0016161616161616162, + "eps": 0, + "loss/policy_avg": -0.23873865604400635, + "loss/value_avg": 8.199307441711426, + "lr": 1.6129032258064518e-07, + "objective/entropy": -73.15070343017578, + "objective/kl": 8.247895240783691, + "objective/non_score_reward": -0.41239476203918457, + "objective/rlhf_reward": 8.626667976379395, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0011615954572334886, + "policy/clipfrac_avg": 0.0034806979820132256, + "policy/entropy_avg": 0.26854443550109863, + "step": 2, + "val/clipfrac_avg": 0.00031833729008212686, + "val/num_eos_tokens": 16, + "val/ratio": 0.9843853712081909, + "val/ratio_var": 0.00014292627747636288 + }, + { + "episode": 48, + "epoch": 0.0024242424242424242, + "eps": 0, + "loss/policy_avg": -0.17912136018276215, + "loss/value_avg": 9.051790237426758, + "lr": 3.2258064516129035e-07, + "objective/entropy": -57.60097885131836, + "objective/kl": 5.790386199951172, + "objective/non_score_reward": -0.2895193099975586, + "objective/rlhf_reward": 7.066926002502441, + "objective/scores": 7.34375, + "policy/approxkl_avg": 0.0009981861803680658, + "policy/clipfrac_avg": 0.003842724720016122, + "policy/entropy_avg": 0.2743988037109375, + "step": 3, + "val/clipfrac_avg": 0.00032501743407920003, + "val/num_eos_tokens": 85, + "val/ratio": 0.9859670400619507, + "val/ratio_var": 7.310912042157724e-05 + }, + { + "episode": 64, + "epoch": 0.0032323232323232323, + "eps": 0, + "loss/policy_avg": -0.11512397229671478, + "loss/value_avg": 6.804945945739746, + "lr": 4.838709677419355e-07, + "objective/entropy": -63.22553634643555, + "objective/kl": 8.576089859008789, + "objective/non_score_reward": -0.42880451679229736, + "objective/rlhf_reward": 7.743070125579834, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.001054076012223959, + "policy/clipfrac_avg": 0.0036023426800966263, + "policy/entropy_avg": 0.2464141845703125, + "step": 4, + "val/clipfrac_avg": 0.0008609364158473909, + "val/num_eos_tokens": 422, + "val/ratio": 0.9846372604370117, + "val/ratio_var": 7.18827432137914e-05 + }, + { + "episode": 80, + "epoch": 0.00404040404040404, + "eps": 0, + "loss/policy_avg": -0.07829581201076508, + "loss/value_avg": 6.955084323883057, + "lr": 6.451612903225807e-07, + "objective/entropy": -114.28401184082031, + "objective/kl": 5.362919807434082, + "objective/non_score_reward": -0.26814600825309753, + "objective/rlhf_reward": 7.38810396194458, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0007407054654322565, + "policy/clipfrac_avg": 0.004495600238442421, + "policy/entropy_avg": 0.16756057739257812, + "step": 5, + "val/clipfrac_avg": 0.0008702859631739557, + "val/num_eos_tokens": 962, + "val/ratio": 0.9889297485351562, + "val/ratio_var": 4.611695112544112e-05 + }, + { + "episode": 96, + "epoch": 0.0048484848484848485, + "eps": 0, + "loss/policy_avg": -0.2765856385231018, + "loss/value_avg": 9.976322174072266, + "lr": 8.064516129032258e-07, + "objective/entropy": -120.7701644897461, + "objective/kl": 5.353013038635254, + "objective/non_score_reward": -0.26765066385269165, + "objective/rlhf_reward": 8.775318145751953, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0007811508257873356, + "policy/clipfrac_avg": 0.0025352579541504383, + "policy/entropy_avg": 0.21323394775390625, + "step": 6, + "val/clipfrac_avg": 0.0010816564317792654, + "val/num_eos_tokens": 128, + "val/ratio": 0.988487720489502, + "val/ratio_var": 6.569296238012612e-05 + }, + { + "episode": 112, + "epoch": 0.0056565656565656566, + "eps": 0, + "loss/policy_avg": -0.036815427243709564, + "loss/value_avg": 5.047958850860596, + "lr": 9.67741935483871e-07, + "objective/entropy": -57.45833969116211, + "objective/kl": 6.929623126983643, + "objective/non_score_reward": -0.3464811444282532, + "objective/rlhf_reward": 7.094925403594971, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0009421511786058545, + "policy/clipfrac_avg": 0.00409944262355566, + "policy/entropy_avg": 0.206146240234375, + "step": 7, + "val/clipfrac_avg": 0.0008602802408859134, + "val/num_eos_tokens": 150, + "val/ratio": 0.9869779348373413, + "val/ratio_var": 3.1514064176008105e-05 + }, + { + "episode": 128, + "epoch": 0.006464646464646465, + "eps": 0, + "loss/policy_avg": -0.13899217545986176, + "loss/value_avg": 12.854679107666016, + "lr": 1.1290322580645162e-06, + "objective/entropy": -4.442367076873779, + "objective/kl": 4.520520210266113, + "objective/non_score_reward": -0.22602598369121552, + "objective/rlhf_reward": 11.03569221496582, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011083385907113552, + "policy/clipfrac_avg": 0.004277850966900587, + "policy/entropy_avg": 0.23640060424804688, + "step": 8, + "val/clipfrac_avg": 0.001243304694071412, + "val/num_eos_tokens": 4, + "val/ratio": 0.9858227968215942, + "val/ratio_var": 6.114065763540566e-05 + }, + { + "episode": 144, + "epoch": 0.007272727272727273, + "eps": 0, + "loss/policy_avg": -0.19064569473266602, + "loss/value_avg": 16.10247802734375, + "lr": 1.2903225806451614e-06, + "objective/entropy": -90.28558349609375, + "objective/kl": 4.24302339553833, + "objective/non_score_reward": -0.2121511697769165, + "objective/rlhf_reward": 8.588630676269531, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0007802502950653434, + "policy/clipfrac_avg": 0.0023238342255353928, + "policy/entropy_avg": 0.16461068391799927, + "step": 9, + "val/clipfrac_avg": 0.0005082917050458491, + "val/num_eos_tokens": 11, + "val/ratio": 0.9889500737190247, + "val/ratio_var": 5.1900708058383316e-05 + }, + { + "episode": 160, + "epoch": 0.00808080808080808, + "eps": 0, + "loss/policy_avg": -0.13336265087127686, + "loss/value_avg": 5.764339923858643, + "lr": 1.4516129032258066e-06, + "objective/entropy": -120.9984359741211, + "objective/kl": 8.206380844116211, + "objective/non_score_reward": -0.41031908988952637, + "objective/rlhf_reward": 6.3904619216918945, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009577925666235387, + "policy/clipfrac_avg": 0.005130666308104992, + "policy/entropy_avg": 0.23892974853515625, + "step": 10, + "val/clipfrac_avg": 0.0019272974459454417, + "val/num_eos_tokens": 368, + "val/ratio": 0.9874762296676636, + "val/ratio_var": 6.36352488072589e-05 + }, + { + "episode": 176, + "epoch": 0.008888888888888889, + "eps": 0, + "loss/policy_avg": -0.1956510990858078, + "loss/value_avg": 9.056394577026367, + "lr": 1.6129032258064516e-06, + "objective/entropy": -47.950660705566406, + "objective/kl": 5.863861560821533, + "objective/non_score_reward": -0.29319310188293457, + "objective/rlhf_reward": 8.073994636535645, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011588437482714653, + "policy/clipfrac_avg": 0.0038244768511503935, + "policy/entropy_avg": 0.2566804885864258, + "step": 11, + "val/clipfrac_avg": 0.0022193677723407745, + "val/num_eos_tokens": 395, + "val/ratio": 0.9839282035827637, + "val/ratio_var": 9.391677303938195e-05 + }, + { + "episode": 192, + "epoch": 0.009696969696969697, + "eps": 0, + "loss/policy_avg": -0.09844052791595459, + "loss/value_avg": 5.998026371002197, + "lr": 1.774193548387097e-06, + "objective/entropy": -56.2935791015625, + "objective/kl": 7.083408832550049, + "objective/non_score_reward": -0.35417044162750244, + "objective/rlhf_reward": 8.083329200744629, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001030836603604257, + "policy/clipfrac_avg": 0.004677055403590202, + "policy/entropy_avg": 0.21318721771240234, + "step": 12, + "val/clipfrac_avg": 0.0010991215240210295, + "val/num_eos_tokens": 36, + "val/ratio": 0.9859715700149536, + "val/ratio_var": 4.26560545747634e-05 + }, + { + "episode": 208, + "epoch": 0.010505050505050505, + "eps": 0, + "loss/policy_avg": -0.10422849655151367, + "loss/value_avg": 7.215424060821533, + "lr": 1.935483870967742e-06, + "objective/entropy": -47.072654724121094, + "objective/kl": 6.387138843536377, + "objective/non_score_reward": -0.31935691833496094, + "objective/rlhf_reward": 6.657205581665039, + "objective/scores": 6.96875, + "policy/approxkl_avg": 0.0011611657682806253, + "policy/clipfrac_avg": 0.005622599273920059, + "policy/entropy_avg": 0.25177001953125, + "step": 13, + "val/clipfrac_avg": 0.005995843559503555, + "val/num_eos_tokens": 541, + "val/ratio": 0.9847086071968079, + "val/ratio_var": 6.94288028171286e-05 + }, + { + "episode": 224, + "epoch": 0.011313131313131313, + "eps": 0, + "loss/policy_avg": -0.12459716200828552, + "loss/value_avg": 7.5816330909729, + "lr": 2.096774193548387e-06, + "objective/entropy": -41.007904052734375, + "objective/kl": 6.544907569885254, + "objective/non_score_reward": -0.3272453844547272, + "objective/rlhf_reward": 8.280176162719727, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001087480690330267, + "policy/clipfrac_avg": 0.00600704550743103, + "policy/entropy_avg": 0.26541900634765625, + "step": 14, + "val/clipfrac_avg": 0.005673446226865053, + "val/num_eos_tokens": 407, + "val/ratio": 0.9857578277587891, + "val/ratio_var": 9.339387906948105e-05 + }, + { + "episode": 240, + "epoch": 0.012121212121212121, + "eps": 0, + "loss/policy_avg": -0.3509424924850464, + "loss/value_avg": 8.651493072509766, + "lr": 2.2580645161290324e-06, + "objective/entropy": -152.62283325195312, + "objective/kl": 6.37682580947876, + "objective/non_score_reward": -0.31884127855300903, + "objective/rlhf_reward": 8.247565269470215, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0008043860434554517, + "policy/clipfrac_avg": 0.004083259496837854, + "policy/entropy_avg": 0.19263720512390137, + "step": 15, + "val/clipfrac_avg": 0.02940819412469864, + "val/num_eos_tokens": 479, + "val/ratio": 0.9888343214988708, + "val/ratio_var": 6.805222801631317e-05 + }, + { + "episode": 256, + "epoch": 0.01292929292929293, + "eps": 0, + "loss/policy_avg": -0.20472919940948486, + "loss/value_avg": 7.683527946472168, + "lr": 2.4193548387096776e-06, + "objective/entropy": -69.35281372070312, + "objective/kl": 7.03504753112793, + "objective/non_score_reward": -0.3517524003982544, + "objective/rlhf_reward": 8.050591468811035, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0011161768343299627, + "policy/clipfrac_avg": 0.005170322954654694, + "policy/entropy_avg": 0.2497091293334961, + "step": 16, + "val/clipfrac_avg": 0.051273077726364136, + "val/num_eos_tokens": 17, + "val/ratio": 0.9848048686981201, + "val/ratio_var": 8.986083412310109e-05 + }, + { + "episode": 272, + "epoch": 0.013737373737373737, + "eps": 0, + "loss/policy_avg": -0.32048213481903076, + "loss/value_avg": 13.413436889648438, + "lr": 2.580645161290323e-06, + "objective/entropy": -107.79791259765625, + "objective/kl": 6.518709659576416, + "objective/non_score_reward": -0.3259354829788208, + "objective/rlhf_reward": 8.931877136230469, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009423322044312954, + "policy/clipfrac_avg": 0.0035665533505380154, + "policy/entropy_avg": 0.19956940412521362, + "step": 17, + "val/clipfrac_avg": 0.066550612449646, + "val/num_eos_tokens": 636, + "val/ratio": 0.9872732162475586, + "val/ratio_var": 6.39159043203108e-05 + }, + { + "episode": 288, + "epoch": 0.014545454545454545, + "eps": 0, + "loss/policy_avg": -0.16326521337032318, + "loss/value_avg": 7.0222063064575195, + "lr": 2.7419354838709676e-06, + "objective/entropy": -60.37420654296875, + "objective/kl": 3.65440034866333, + "objective/non_score_reward": -0.18272003531455994, + "objective/rlhf_reward": 7.470600605010986, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0009109527454711497, + "policy/clipfrac_avg": 0.0032424808014184237, + "policy/entropy_avg": 0.21929168701171875, + "step": 18, + "val/clipfrac_avg": 0.15316107869148254, + "val/num_eos_tokens": 376, + "val/ratio": 0.9875004291534424, + "val/ratio_var": 6.943759944988415e-05 + }, + { + "episode": 304, + "epoch": 0.015353535353535354, + "eps": 0, + "loss/policy_avg": -0.09499570727348328, + "loss/value_avg": 4.847567558288574, + "lr": 2.903225806451613e-06, + "objective/entropy": -64.48834228515625, + "objective/kl": 8.19035530090332, + "objective/non_score_reward": -0.40951770544052124, + "objective/rlhf_reward": 7.082670211791992, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0009629215346649289, + "policy/clipfrac_avg": 0.004428057000041008, + "policy/entropy_avg": 0.21021270751953125, + "step": 19, + "val/clipfrac_avg": 0.09573161602020264, + "val/num_eos_tokens": 161, + "val/ratio": 0.9871885776519775, + "val/ratio_var": 5.789758506580256e-05 + }, + { + "episode": 320, + "epoch": 0.01616161616161616, + "eps": 0, + "loss/policy_avg": -0.36197370290756226, + "loss/value_avg": 9.9912109375, + "lr": 3.0645161290322584e-06, + "objective/entropy": -92.24372863769531, + "objective/kl": 4.762131690979004, + "objective/non_score_reward": -0.23810657858848572, + "objective/rlhf_reward": 8.150565147399902, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0009718802757561207, + "policy/clipfrac_avg": 0.004476544447243214, + "policy/entropy_avg": 0.261799693107605, + "step": 20, + "val/clipfrac_avg": 0.15394406020641327, + "val/num_eos_tokens": 306, + "val/ratio": 0.9875181317329407, + "val/ratio_var": 9.373524517286569e-05 + }, + { + "episode": 336, + "epoch": 0.01696969696969697, + "eps": 0, + "loss/policy_avg": -0.06269250065088272, + "loss/value_avg": 5.039106369018555, + "lr": 3.225806451612903e-06, + "objective/entropy": -60.66798400878906, + "objective/kl": 5.371654510498047, + "objective/non_score_reward": -0.26858270168304443, + "objective/rlhf_reward": 6.957979679107666, + "objective/scores": 7.21875, + "policy/approxkl_avg": 0.0009720490779727697, + "policy/clipfrac_avg": 0.002186079975217581, + "policy/entropy_avg": 0.2316131591796875, + "step": 21, + "val/clipfrac_avg": 0.11872587352991104, + "val/num_eos_tokens": 62, + "val/ratio": 0.9866783618927002, + "val/ratio_var": 5.079948095954023e-05 + }, + { + "episode": 352, + "epoch": 0.017777777777777778, + "eps": 0, + "loss/policy_avg": -0.1775662899017334, + "loss/value_avg": 9.879995346069336, + "lr": 3.3870967741935484e-06, + "objective/entropy": -52.22957229614258, + "objective/kl": 3.4860763549804688, + "objective/non_score_reward": -0.1743038147687912, + "objective/rlhf_reward": 8.28663444519043, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007850549882277846, + "policy/clipfrac_avg": 0.004184068646281958, + "policy/entropy_avg": 0.17317485809326172, + "step": 22, + "val/clipfrac_avg": 0.19386646151542664, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883713722229004, + "val/ratio_var": 5.718659667763859e-05 + }, + { + "episode": 368, + "epoch": 0.018585858585858588, + "eps": 0, + "loss/policy_avg": -0.08608929812908173, + "loss/value_avg": 5.026681900024414, + "lr": 3.548387096774194e-06, + "objective/entropy": -17.842317581176758, + "objective/kl": 6.329047203063965, + "objective/non_score_reward": -0.31645238399505615, + "objective/rlhf_reward": 9.144485473632812, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001143642351962626, + "policy/clipfrac_avg": 0.0046859681606292725, + "policy/entropy_avg": 0.245330810546875, + "step": 23, + "val/clipfrac_avg": 0.11751493066549301, + "val/num_eos_tokens": 412, + "val/ratio": 0.9848726987838745, + "val/ratio_var": 6.24043750576675e-05 + }, + { + "episode": 384, + "epoch": 0.019393939393939394, + "eps": 0, + "loss/policy_avg": -0.010797667317092419, + "loss/value_avg": 6.275961875915527, + "lr": 3.7096774193548392e-06, + "objective/entropy": -47.785430908203125, + "objective/kl": 5.16103458404541, + "objective/non_score_reward": -0.2580517530441284, + "objective/rlhf_reward": 7.452885627746582, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009465434122830629, + "policy/clipfrac_avg": 0.003977151587605476, + "policy/entropy_avg": 0.28936767578125, + "step": 24, + "val/clipfrac_avg": 0.12968723475933075, + "val/num_eos_tokens": 200, + "val/ratio": 0.9866119027137756, + "val/ratio_var": 6.631347787333652e-05 + }, + { + "episode": 400, + "epoch": 0.020202020202020204, + "eps": 0, + "loss/policy_avg": 0.03891272097826004, + "loss/value_avg": 5.366772651672363, + "lr": 3.870967741935484e-06, + "objective/entropy": -20.867111206054688, + "objective/kl": 7.756465911865234, + "objective/non_score_reward": -0.38782334327697754, + "objective/rlhf_reward": 7.920770645141602, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012229372514411807, + "policy/clipfrac_avg": 0.0030088755302131176, + "policy/entropy_avg": 0.2554168701171875, + "step": 25, + "val/clipfrac_avg": 0.12619256973266602, + "val/num_eos_tokens": 16, + "val/ratio": 0.9858287572860718, + "val/ratio_var": 6.279556691879407e-05 + }, + { + "episode": 416, + "epoch": 0.02101010101010101, + "eps": 0, + "loss/policy_avg": -0.09346847981214523, + "loss/value_avg": 3.7916016578674316, + "lr": 4.032258064516129e-06, + "objective/entropy": -131.1887969970703, + "objective/kl": 6.121448040008545, + "objective/non_score_reward": -0.3060724139213562, + "objective/rlhf_reward": 7.201740264892578, + "objective/scores": 7.5, + "policy/approxkl_avg": 0.0008630367228761315, + "policy/clipfrac_avg": 0.003512686351314187, + "policy/entropy_avg": 0.2108306884765625, + "step": 26, + "val/clipfrac_avg": 0.10546709597110748, + "val/num_eos_tokens": 212, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 6.414195377146825e-05 + }, + { + "episode": 432, + "epoch": 0.02181818181818182, + "eps": 0, + "loss/policy_avg": -0.08439041674137115, + "loss/value_avg": 4.865840435028076, + "lr": 4.193548387096774e-06, + "objective/entropy": -91.82192993164062, + "objective/kl": 6.764763355255127, + "objective/non_score_reward": -0.3382381796836853, + "objective/rlhf_reward": 9.751605987548828, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.000947782420553267, + "policy/clipfrac_avg": 0.005451520439237356, + "policy/entropy_avg": 0.209228515625, + "step": 27, + "val/clipfrac_avg": 0.06509673595428467, + "val/num_eos_tokens": 221, + "val/ratio": 0.9870284795761108, + "val/ratio_var": 4.723674283013679e-05 + }, + { + "episode": 448, + "epoch": 0.022626262626262626, + "eps": 0, + "loss/policy_avg": -0.12459178268909454, + "loss/value_avg": 4.973685264587402, + "lr": 4.35483870967742e-06, + "objective/entropy": -68.857421875, + "objective/kl": 7.102960586547852, + "objective/non_score_reward": -0.3551480770111084, + "objective/rlhf_reward": 8.937820434570312, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010395422577857971, + "policy/clipfrac_avg": 0.005196031183004379, + "policy/entropy_avg": 0.22506141662597656, + "step": 28, + "val/clipfrac_avg": 0.09726891666650772, + "val/num_eos_tokens": 153, + "val/ratio": 0.9859325885772705, + "val/ratio_var": 7.372920663328841e-05 + }, + { + "episode": 464, + "epoch": 0.023434343434343436, + "eps": 0, + "loss/policy_avg": -0.0893494039773941, + "loss/value_avg": 4.280435085296631, + "lr": 4.516129032258065e-06, + "objective/entropy": -39.28635787963867, + "objective/kl": 4.585806846618652, + "objective/non_score_reward": -0.22929036617279053, + "objective/rlhf_reward": 7.24336576461792, + "objective/scores": 7.46875, + "policy/approxkl_avg": 0.0010784105397760868, + "policy/clipfrac_avg": 0.004656917415559292, + "policy/entropy_avg": 0.28299713134765625, + "step": 29, + "val/clipfrac_avg": 0.033933281898498535, + "val/num_eos_tokens": 151, + "val/ratio": 0.9854025840759277, + "val/ratio_var": 7.980425289133564e-05 + }, + { + "episode": 480, + "epoch": 0.024242424242424242, + "eps": 0, + "loss/policy_avg": 0.0009674776811152697, + "loss/value_avg": 3.2916879653930664, + "lr": 4.67741935483871e-06, + "objective/entropy": -56.16079330444336, + "objective/kl": 3.2583465576171875, + "objective/non_score_reward": -0.1629173308610916, + "objective/rlhf_reward": 6.471848487854004, + "objective/scores": 6.625, + "policy/approxkl_avg": 0.0009038803982548416, + "policy/clipfrac_avg": 0.0039865439757704735, + "policy/entropy_avg": 0.21944808959960938, + "step": 30, + "val/clipfrac_avg": 0.037767596542835236, + "val/num_eos_tokens": 229, + "val/ratio": 0.9880707263946533, + "val/ratio_var": 7.166655996115878e-05 + }, + { + "episode": 496, + "epoch": 0.025050505050505052, + "eps": 0, + "loss/policy_avg": -0.05671697109937668, + "loss/value_avg": 4.5554399490356445, + "lr": 4.838709677419355e-06, + "objective/entropy": -85.58920288085938, + "objective/kl": 4.62398624420166, + "objective/non_score_reward": -0.23119933903217316, + "objective/rlhf_reward": 8.104738235473633, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0008234091801568866, + "policy/clipfrac_avg": 0.0037510073743760586, + "policy/entropy_avg": 0.18961238861083984, + "step": 31, + "val/clipfrac_avg": 0.12869636714458466, + "val/num_eos_tokens": 30, + "val/ratio": 0.9884214997291565, + "val/ratio_var": 7.101450319169089e-05 + }, + { + "episode": 512, + "epoch": 0.02585858585858586, + "eps": 0, + "loss/policy_avg": -0.08824437111616135, + "loss/value_avg": 3.4218335151672363, + "lr": 5e-06, + "objective/entropy": -81.65757751464844, + "objective/kl": 5.964879035949707, + "objective/non_score_reward": -0.2982439696788788, + "objective/rlhf_reward": 6.811131000518799, + "objective/scores": 7.125, + "policy/approxkl_avg": 0.0008165108738467097, + "policy/clipfrac_avg": 0.0019289799965918064, + "policy/entropy_avg": 0.1952981948852539, + "step": 32, + "val/clipfrac_avg": 0.10708418488502502, + "val/num_eos_tokens": 82, + "val/ratio": 0.9881972074508667, + "val/ratio_var": 7.150996680138633e-05 + }, + { + "episode": 528, + "epoch": 0.02666666666666667, + "eps": 0, + "loss/policy_avg": 0.09306781738996506, + "loss/value_avg": 3.8982434272766113, + "lr": 5.161290322580646e-06, + "objective/entropy": -73.17301177978516, + "objective/kl": 3.579554557800293, + "objective/non_score_reward": -0.17897772789001465, + "objective/rlhf_reward": 7.766334533691406, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0008378748898394406, + "policy/clipfrac_avg": 0.0022445092909038067, + "policy/entropy_avg": 0.22289752960205078, + "step": 33, + "val/clipfrac_avg": 0.09437251091003418, + "val/num_eos_tokens": 388, + "val/ratio": 0.9860420227050781, + "val/ratio_var": 6.518009467981756e-05 + }, + { + "episode": 544, + "epoch": 0.027474747474747475, + "eps": 0, + "loss/policy_avg": -0.2350362241268158, + "loss/value_avg": 5.297033309936523, + "lr": 5.322580645161291e-06, + "objective/entropy": -104.75776672363281, + "objective/kl": 5.92071533203125, + "objective/non_score_reward": -0.2960357964038849, + "objective/rlhf_reward": 8.387557983398438, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.001011505490168929, + "policy/clipfrac_avg": 0.004968347493559122, + "policy/entropy_avg": 0.22486305236816406, + "step": 34, + "val/clipfrac_avg": 0.33360281586647034, + "val/num_eos_tokens": 871, + "val/ratio": 0.9869726300239563, + "val/ratio_var": 0.00013502036745194346 + }, + { + "episode": 560, + "epoch": 0.028282828282828285, + "eps": 0, + "loss/policy_avg": 0.01827317103743553, + "loss/value_avg": 2.6717967987060547, + "lr": 5.483870967741935e-06, + "objective/entropy": -92.61628723144531, + "objective/kl": 5.674022674560547, + "objective/non_score_reward": -0.2837011516094208, + "objective/rlhf_reward": 6.544424057006836, + "objective/scores": 6.8125, + "policy/approxkl_avg": 0.0009126091026701033, + "policy/clipfrac_avg": 0.005444251000881195, + "policy/entropy_avg": 0.2899208068847656, + "step": 35, + "val/clipfrac_avg": 0.19933006167411804, + "val/num_eos_tokens": 494, + "val/ratio": 0.9873366355895996, + "val/ratio_var": 0.00010928975098067895 + }, + { + "episode": 576, + "epoch": 0.02909090909090909, + "eps": 0, + "loss/policy_avg": -0.45309978723526, + "loss/value_avg": 7.705583572387695, + "lr": 5.645161290322582e-06, + "objective/entropy": -116.77957916259766, + "objective/kl": 5.049452304840088, + "objective/non_score_reward": -0.2524726092815399, + "objective/rlhf_reward": 11.341278076171875, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0007875356823205948, + "policy/clipfrac_avg": 0.002243435475975275, + "policy/entropy_avg": 0.1687488555908203, + "step": 36, + "val/clipfrac_avg": 0.41723158955574036, + "val/num_eos_tokens": 472, + "val/ratio": 0.9891025424003601, + "val/ratio_var": 7.570298475911841e-05 + }, + { + "episode": 592, + "epoch": 0.0298989898989899, + "eps": 0, + "loss/policy_avg": -0.27262574434280396, + "loss/value_avg": 4.820296287536621, + "lr": 5.806451612903226e-06, + "objective/entropy": -141.64932250976562, + "objective/kl": 6.886253833770752, + "objective/non_score_reward": -0.3443126678466797, + "objective/rlhf_reward": 8.91349983215332, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009149812394753098, + "policy/clipfrac_avg": 0.0036448687314987183, + "policy/entropy_avg": 0.20346355438232422, + "step": 37, + "val/clipfrac_avg": 0.12755423784255981, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880065321922302, + "val/ratio_var": 0.00010917898180196062 + }, + { + "episode": 608, + "epoch": 0.030707070707070707, + "eps": 0, + "loss/policy_avg": -0.34780561923980713, + "loss/value_avg": 6.757951736450195, + "lr": 5.967741935483872e-06, + "objective/entropy": -68.97830963134766, + "objective/kl": 4.568058967590332, + "objective/non_score_reward": -0.2284029722213745, + "objective/rlhf_reward": 8.593862533569336, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0009708706638775766, + "policy/clipfrac_avg": 0.003779984312132001, + "policy/entropy_avg": 0.23283910751342773, + "step": 38, + "val/clipfrac_avg": 0.19302648305892944, + "val/num_eos_tokens": 504, + "val/ratio": 0.9863685369491577, + "val/ratio_var": 0.00010391126124886796 + }, + { + "episode": 624, + "epoch": 0.03151515151515152, + "eps": 0, + "loss/policy_avg": -0.19798313081264496, + "loss/value_avg": 3.1714487075805664, + "lr": 6.129032258064517e-06, + "objective/entropy": -96.96963500976562, + "objective/kl": 4.91707181930542, + "objective/non_score_reward": -0.24585360288619995, + "objective/rlhf_reward": 8.472895622253418, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008412165334448218, + "policy/clipfrac_avg": 0.0068719531409442425, + "policy/entropy_avg": 0.1824016571044922, + "step": 39, + "val/clipfrac_avg": 0.14115071296691895, + "val/num_eos_tokens": 150, + "val/ratio": 0.9887563586235046, + "val/ratio_var": 7.865983934607357e-05 + }, + { + "episode": 640, + "epoch": 0.03232323232323232, + "eps": 0, + "loss/policy_avg": -0.18319812417030334, + "loss/value_avg": 2.141171455383301, + "lr": 6.290322580645162e-06, + "objective/entropy": -71.3642807006836, + "objective/kl": 6.146427631378174, + "objective/non_score_reward": -0.3073214292526245, + "objective/rlhf_reward": 8.700490951538086, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0008069784380495548, + "policy/clipfrac_avg": 0.004310991615056992, + "policy/entropy_avg": 0.1599130630493164, + "step": 40, + "val/clipfrac_avg": 0.11660860478878021, + "val/num_eos_tokens": 20, + "val/ratio": 0.9895697832107544, + "val/ratio_var": 3.526661384967156e-05 + }, + { + "episode": 656, + "epoch": 0.03313131313131313, + "eps": 0, + "loss/policy_avg": -0.050840627402067184, + "loss/value_avg": 3.107524871826172, + "lr": 6.451612903225806e-06, + "objective/entropy": -72.12484741210938, + "objective/kl": 6.445234775543213, + "objective/non_score_reward": -0.3222617506980896, + "objective/rlhf_reward": 9.904300689697266, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.001101147965528071, + "policy/clipfrac_avg": 0.005892510525882244, + "policy/entropy_avg": 0.23779296875, + "step": 41, + "val/clipfrac_avg": 0.28902360796928406, + "val/num_eos_tokens": 54, + "val/ratio": 0.9857878684997559, + "val/ratio_var": 8.489257743349299e-05 + }, + { + "episode": 672, + "epoch": 0.03393939393939394, + "eps": 0, + "loss/policy_avg": -0.12600231170654297, + "loss/value_avg": 4.626214981079102, + "lr": 6.612903225806452e-06, + "objective/entropy": -90.39147186279297, + "objective/kl": 7.493732452392578, + "objective/non_score_reward": -0.3746866285800934, + "objective/rlhf_reward": 6.710762977600098, + "objective/scores": 7.09375, + "policy/approxkl_avg": 0.0008206125348806381, + "policy/clipfrac_avg": 0.002678059274330735, + "policy/entropy_avg": 0.19233226776123047, + "step": 42, + "val/clipfrac_avg": 0.11169028282165527, + "val/num_eos_tokens": 8, + "val/ratio": 0.988521933555603, + "val/ratio_var": 8.11065110610798e-05 + }, + { + "episode": 688, + "epoch": 0.03474747474747475, + "eps": 0, + "loss/policy_avg": -0.43953514099121094, + "loss/value_avg": 6.373354911804199, + "lr": 6.774193548387097e-06, + "objective/entropy": -97.91592407226562, + "objective/kl": 5.224013328552246, + "objective/non_score_reward": -0.2612006962299347, + "objective/rlhf_reward": 9.063018798828125, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010877563618123531, + "policy/clipfrac_avg": 0.004649983253329992, + "policy/entropy_avg": 0.2086353302001953, + "step": 43, + "val/clipfrac_avg": 0.22453558444976807, + "val/num_eos_tokens": 302, + "val/ratio": 0.985620379447937, + "val/ratio_var": 0.0001359783491352573 + }, + { + "episode": 704, + "epoch": 0.035555555555555556, + "eps": 0, + "loss/policy_avg": -0.2054598033428192, + "loss/value_avg": 3.1007838249206543, + "lr": 6.935483870967743e-06, + "objective/entropy": -113.148681640625, + "objective/kl": 8.199766159057617, + "objective/non_score_reward": -0.4099883437156677, + "objective/rlhf_reward": 6.9806365966796875, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0009478515712544322, + "policy/clipfrac_avg": 0.002450365573167801, + "policy/entropy_avg": 0.21928471326828003, + "step": 44, + "val/clipfrac_avg": 0.11797119677066803, + "val/num_eos_tokens": 408, + "val/ratio": 0.9870466589927673, + "val/ratio_var": 9.353987115900964e-05 + }, + { + "episode": 720, + "epoch": 0.03636363636363636, + "eps": 0, + "loss/policy_avg": 0.06887191534042358, + "loss/value_avg": 1.435150384902954, + "lr": 7.096774193548388e-06, + "objective/entropy": -61.80511474609375, + "objective/kl": 6.225659370422363, + "objective/non_score_reward": -0.3112829923629761, + "objective/rlhf_reward": 7.524654388427734, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009408287587575614, + "policy/clipfrac_avg": 0.00496808672323823, + "policy/entropy_avg": 0.19115066528320312, + "step": 45, + "val/clipfrac_avg": 0.1740478277206421, + "val/num_eos_tokens": 165, + "val/ratio": 0.9886248111724854, + "val/ratio_var": 4.797361543751322e-05 + }, + { + "episode": 736, + "epoch": 0.037171717171717175, + "eps": 0, + "loss/policy_avg": -0.008727293461561203, + "loss/value_avg": 1.2011616230010986, + "lr": 7.258064516129033e-06, + "objective/entropy": -80.8755111694336, + "objective/kl": 9.153085708618164, + "objective/non_score_reward": -0.45765426754951477, + "objective/rlhf_reward": 7.393908500671387, + "objective/scores": 7.84375, + "policy/approxkl_avg": 0.0009702641982585192, + "policy/clipfrac_avg": 0.0038732639513909817, + "policy/entropy_avg": 0.20413970947265625, + "step": 46, + "val/clipfrac_avg": 0.1519191861152649, + "val/num_eos_tokens": 330, + "val/ratio": 0.9864258170127869, + "val/ratio_var": 5.902982229599729e-05 + }, + { + "episode": 752, + "epoch": 0.03797979797979798, + "eps": 0, + "loss/policy_avg": -0.009945835918188095, + "loss/value_avg": 1.0838792324066162, + "lr": 7.4193548387096784e-06, + "objective/entropy": -74.29957580566406, + "objective/kl": 7.737156867980957, + "objective/non_score_reward": -0.38685786724090576, + "objective/rlhf_reward": 7.066267013549805, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.0008943852735683322, + "policy/clipfrac_avg": 0.007206573151051998, + "policy/entropy_avg": 0.1891345977783203, + "step": 47, + "val/clipfrac_avg": 0.15996162593364716, + "val/num_eos_tokens": 539, + "val/ratio": 0.9879242181777954, + "val/ratio_var": 4.614832141669467e-05 + }, + { + "episode": 768, + "epoch": 0.03878787878787879, + "eps": 0, + "loss/policy_avg": -0.12174442410469055, + "loss/value_avg": 4.897049903869629, + "lr": 7.580645161290323e-06, + "objective/entropy": -73.60452270507812, + "objective/kl": 4.253904819488525, + "objective/non_score_reward": -0.21269527077674866, + "objective/rlhf_reward": 7.465039253234863, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.001011924003250897, + "policy/clipfrac_avg": 0.01086447387933731, + "policy/entropy_avg": 0.2534313201904297, + "step": 48, + "val/clipfrac_avg": 0.2783915400505066, + "val/num_eos_tokens": 180, + "val/ratio": 0.9866998195648193, + "val/ratio_var": 0.0001104455950553529 + }, + { + "episode": 784, + "epoch": 0.039595959595959594, + "eps": 0, + "loss/policy_avg": -0.39860063791275024, + "loss/value_avg": 3.1691648960113525, + "lr": 7.741935483870968e-06, + "objective/entropy": -61.99099349975586, + "objective/kl": 6.293693542480469, + "objective/non_score_reward": -0.3146846890449524, + "objective/rlhf_reward": 7.892346382141113, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0010794082190841436, + "policy/clipfrac_avg": 0.013865058310329914, + "policy/entropy_avg": 0.21884393692016602, + "step": 49, + "val/clipfrac_avg": 0.16114407777786255, + "val/num_eos_tokens": 13, + "val/ratio": 0.9858240485191345, + "val/ratio_var": 0.00017287657829001546 + }, + { + "episode": 800, + "epoch": 0.04040404040404041, + "eps": 0, + "loss/policy_avg": -0.018761441111564636, + "loss/value_avg": 1.6952381134033203, + "lr": 7.903225806451613e-06, + "objective/entropy": -71.7900390625, + "objective/kl": 6.167643070220947, + "objective/non_score_reward": -0.30838215351104736, + "objective/rlhf_reward": 7.576383590698242, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.000964264792855829, + "policy/clipfrac_avg": 0.009008180350065231, + "policy/entropy_avg": 0.24611949920654297, + "step": 50, + "val/clipfrac_avg": 0.14082017540931702, + "val/num_eos_tokens": 98, + "val/ratio": 0.9882640838623047, + "val/ratio_var": 6.893485260661691e-05 + }, + { + "episode": 816, + "epoch": 0.041212121212121214, + "eps": 0, + "loss/policy_avg": -0.045411769300699234, + "loss/value_avg": 1.3062368631362915, + "lr": 8.064516129032258e-06, + "objective/entropy": -79.89970397949219, + "objective/kl": 5.655811309814453, + "objective/non_score_reward": -0.28279057145118713, + "objective/rlhf_reward": 7.611741065979004, + "objective/scores": 7.90625, + "policy/approxkl_avg": 0.0008801341755315661, + "policy/clipfrac_avg": 0.007216072641313076, + "policy/entropy_avg": 0.18111515045166016, + "step": 51, + "val/clipfrac_avg": 0.0781465545296669, + "val/num_eos_tokens": 21, + "val/ratio": 0.9885283708572388, + "val/ratio_var": 8.526279998477548e-05 + }, + { + "episode": 832, + "epoch": 0.04202020202020202, + "eps": 0, + "loss/policy_avg": -0.14512324333190918, + "loss/value_avg": 1.5389004945755005, + "lr": 8.225806451612904e-06, + "objective/entropy": -170.48452758789062, + "objective/kl": 5.510902404785156, + "objective/non_score_reward": -0.2755451202392578, + "objective/rlhf_reward": 7.029630661010742, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0006108771194703877, + "policy/clipfrac_avg": 0.004314820747822523, + "policy/entropy_avg": 0.1314830780029297, + "step": 52, + "val/clipfrac_avg": 0.08058575540781021, + "val/num_eos_tokens": 337, + "val/ratio": 0.9919232130050659, + "val/ratio_var": 4.701551370089874e-05 + }, + { + "episode": 848, + "epoch": 0.042828282828282827, + "eps": 0, + "loss/policy_avg": -0.38281670212745667, + "loss/value_avg": 3.0069656372070312, + "lr": 8.387096774193549e-06, + "objective/entropy": -96.16239166259766, + "objective/kl": 6.628139495849609, + "objective/non_score_reward": -0.33140695095062256, + "objective/rlhf_reward": 9.07874870300293, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008359593921341002, + "policy/clipfrac_avg": 0.005661661736667156, + "policy/entropy_avg": 0.1713862419128418, + "step": 53, + "val/clipfrac_avg": 0.1024901270866394, + "val/num_eos_tokens": 9, + "val/ratio": 0.9893118143081665, + "val/ratio_var": 6.531424878630787e-05 + }, + { + "episode": 864, + "epoch": 0.04363636363636364, + "eps": 0, + "loss/policy_avg": -0.015762103721499443, + "loss/value_avg": 1.7061840295791626, + "lr": 8.548387096774194e-06, + "objective/entropy": -79.8026123046875, + "objective/kl": 8.291646957397461, + "objective/non_score_reward": -0.41458237171173096, + "objective/rlhf_reward": 9.077606201171875, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009487073984928429, + "policy/clipfrac_avg": 0.027981357648968697, + "policy/entropy_avg": 0.22313880920410156, + "step": 54, + "val/clipfrac_avg": 0.12049607932567596, + "val/num_eos_tokens": 151, + "val/ratio": 0.9876121282577515, + "val/ratio_var": 7.097758498275653e-05 + }, + { + "episode": 880, + "epoch": 0.044444444444444446, + "eps": 0, + "loss/policy_avg": -0.33922648429870605, + "loss/value_avg": 1.7691230773925781, + "lr": 8.70967741935484e-06, + "objective/entropy": -34.682456970214844, + "objective/kl": 10.053032875061035, + "objective/non_score_reward": -0.5026516914367676, + "objective/rlhf_reward": 9.54031753540039, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0009677361231297255, + "policy/clipfrac_avg": 0.0033034570515155792, + "policy/entropy_avg": 0.1980457305908203, + "step": 55, + "val/clipfrac_avg": 0.17926591634750366, + "val/num_eos_tokens": 45, + "val/ratio": 0.9871523380279541, + "val/ratio_var": 4.744104444398545e-05 + }, + { + "episode": 896, + "epoch": 0.04525252525252525, + "eps": 0, + "loss/policy_avg": -0.1486111581325531, + "loss/value_avg": 3.018749713897705, + "lr": 8.870967741935484e-06, + "objective/entropy": -89.58358764648438, + "objective/kl": 7.366510391235352, + "objective/non_score_reward": -0.36832550168037415, + "objective/rlhf_reward": 7.397299289703369, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0008684438071213663, + "policy/clipfrac_avg": 0.012218687683343887, + "policy/entropy_avg": 0.19923949241638184, + "step": 56, + "val/clipfrac_avg": 0.1270667314529419, + "val/num_eos_tokens": 282, + "val/ratio": 0.9883291721343994, + "val/ratio_var": 7.791651296429336e-05 + }, + { + "episode": 912, + "epoch": 0.04606060606060606, + "eps": 0, + "loss/policy_avg": 0.2920738160610199, + "loss/value_avg": 2.8446502685546875, + "lr": 9.03225806451613e-06, + "objective/entropy": -50.028282165527344, + "objective/kl": 9.809646606445312, + "objective/non_score_reward": -0.49048230051994324, + "objective/rlhf_reward": 7.368892669677734, + "objective/scores": 7.875, + "policy/approxkl_avg": 0.0011404260294511914, + "policy/clipfrac_avg": 0.009378134272992611, + "policy/entropy_avg": 0.32576751708984375, + "step": 57, + "val/clipfrac_avg": 0.22250550985336304, + "val/num_eos_tokens": 524, + "val/ratio": 0.9856222867965698, + "val/ratio_var": 4.662256105802953e-05 + }, + { + "episode": 928, + "epoch": 0.04686868686868687, + "eps": 0, + "loss/policy_avg": -0.4045873284339905, + "loss/value_avg": 2.1512539386749268, + "lr": 9.193548387096775e-06, + "objective/entropy": -113.26528930664062, + "objective/kl": 8.777005195617676, + "objective/non_score_reward": -0.4388502836227417, + "objective/rlhf_reward": 9.412712097167969, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008379302453249693, + "policy/clipfrac_avg": 0.012378448620438576, + "policy/entropy_avg": 0.17434120178222656, + "step": 58, + "val/clipfrac_avg": 0.29461991786956787, + "val/num_eos_tokens": 164, + "val/ratio": 0.988739550113678, + "val/ratio_var": 8.456465729977936e-05 + }, + { + "episode": 944, + "epoch": 0.04767676767676768, + "eps": 0, + "loss/policy_avg": 0.15078622102737427, + "loss/value_avg": 1.8014607429504395, + "lr": 9.35483870967742e-06, + "objective/entropy": -101.6314926147461, + "objective/kl": 8.044000625610352, + "objective/non_score_reward": -0.40220004320144653, + "objective/rlhf_reward": 8.168111801147461, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.009142342954874039, + "policy/clipfrac_avg": 0.02357684075832367, + "policy/entropy_avg": 0.19042015075683594, + "step": 59, + "val/clipfrac_avg": 0.2494688183069229, + "val/num_eos_tokens": 741, + "val/ratio": 0.9888198375701904, + "val/ratio_var": 4.30569052696228e-05 + }, + { + "episode": 960, + "epoch": 0.048484848484848485, + "eps": 0, + "loss/policy_avg": 0.05019530653953552, + "loss/value_avg": 2.0129008293151855, + "lr": 9.516129032258065e-06, + "objective/entropy": -105.13027954101562, + "objective/kl": 10.100278854370117, + "objective/non_score_reward": -0.5050139427185059, + "objective/rlhf_reward": 8.147329330444336, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012912012171000242, + "policy/clipfrac_avg": 0.01893148198723793, + "policy/entropy_avg": 0.21358871459960938, + "step": 60, + "val/clipfrac_avg": 0.12193797528743744, + "val/num_eos_tokens": 80, + "val/ratio": 0.9880148768424988, + "val/ratio_var": 6.140353070804849e-05 + }, + { + "episode": 976, + "epoch": 0.04929292929292929, + "eps": 0, + "loss/policy_avg": -0.14436417818069458, + "loss/value_avg": 2.7013297080993652, + "lr": 9.67741935483871e-06, + "objective/entropy": -46.50836944580078, + "objective/kl": 7.407527923583984, + "objective/non_score_reward": -0.3703763782978058, + "objective/rlhf_reward": 9.981185913085938, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0010337126441299915, + "policy/clipfrac_avg": 0.014061791822314262, + "policy/entropy_avg": 0.18859481811523438, + "step": 61, + "val/clipfrac_avg": 0.17322519421577454, + "val/num_eos_tokens": 9, + "val/ratio": 0.9879202246665955, + "val/ratio_var": 5.444034104584716e-05 + }, + { + "episode": 992, + "epoch": 0.050101010101010104, + "eps": 0, + "loss/policy_avg": -0.4379100501537323, + "loss/value_avg": 2.9977834224700928, + "lr": 9.838709677419356e-06, + "objective/entropy": -104.4605712890625, + "objective/kl": 6.459438323974609, + "objective/non_score_reward": -0.3229719400405884, + "objective/rlhf_reward": 10.25124740600586, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008976229000836611, + "policy/clipfrac_avg": 0.00845531839877367, + "policy/entropy_avg": 0.18575620651245117, + "step": 62, + "val/clipfrac_avg": 0.25087857246398926, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883289337158203, + "val/ratio_var": 0.00015057438577059656 + }, + { + "episode": 1008, + "epoch": 0.05090909090909091, + "eps": 0, + "loss/policy_avg": 0.0116458460688591, + "loss/value_avg": 0.6588444709777832, + "lr": 1e-05, + "objective/entropy": -74.31800842285156, + "objective/kl": 8.997238159179688, + "objective/non_score_reward": -0.44986194372177124, + "objective/rlhf_reward": 8.124356269836426, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0009533560369163752, + "policy/clipfrac_avg": 0.01315313670784235, + "policy/entropy_avg": 0.20091533660888672, + "step": 63, + "val/clipfrac_avg": 0.19718438386917114, + "val/num_eos_tokens": 181, + "val/ratio": 0.987379789352417, + "val/ratio_var": 5.34383361809887e-05 + }, + { + "episode": 1024, + "epoch": 0.05171717171717172, + "eps": 0, + "loss/policy_avg": -0.17724251747131348, + "loss/value_avg": 2.510040521621704, + "lr": 9.999982158767522e-06, + "objective/entropy": -115.92865753173828, + "objective/kl": 9.876341819763184, + "objective/non_score_reward": -0.4938170909881592, + "objective/rlhf_reward": 8.553058624267578, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0009387531317770481, + "policy/clipfrac_avg": 0.011313371360301971, + "policy/entropy_avg": 0.2193007469177246, + "step": 64, + "val/clipfrac_avg": 0.19327464699745178, + "val/num_eos_tokens": 433, + "val/ratio": 0.9880937337875366, + "val/ratio_var": 6.919403676874936e-05 + }, + { + "episode": 1040, + "epoch": 0.052525252525252523, + "eps": 0, + "loss/policy_avg": -0.01736600697040558, + "loss/value_avg": 1.1105473041534424, + "lr": 9.999928635197409e-06, + "objective/entropy": -151.23590087890625, + "objective/kl": 12.105890274047852, + "objective/non_score_reward": -0.6052945256233215, + "objective/rlhf_reward": 7.050955295562744, + "objective/scores": 7.65625, + "policy/approxkl_avg": 0.0008311746641993523, + "policy/clipfrac_avg": 0.021521538496017456, + "policy/entropy_avg": 0.16837120056152344, + "step": 65, + "val/clipfrac_avg": 0.2209428995847702, + "val/num_eos_tokens": 350, + "val/ratio": 0.9889916777610779, + "val/ratio_var": 6.483616743935272e-05 + }, + { + "episode": 1056, + "epoch": 0.05333333333333334, + "eps": 0, + "loss/policy_avg": -0.16295818984508514, + "loss/value_avg": 1.7638646364212036, + "lr": 9.999839429671632e-06, + "objective/entropy": -121.24343872070312, + "objective/kl": 9.621443748474121, + "objective/non_score_reward": -0.48107218742370605, + "objective/rlhf_reward": 7.540412425994873, + "objective/scores": 8.0, + "policy/approxkl_avg": 0.0009060403681360185, + "policy/clipfrac_avg": 0.010464828461408615, + "policy/entropy_avg": 0.1780584454536438, + "step": 66, + "val/clipfrac_avg": 0.20382188260555267, + "val/num_eos_tokens": 91, + "val/ratio": 0.9872162342071533, + "val/ratio_var": 0.00010221501725027338 + }, + { + "episode": 1072, + "epoch": 0.05414141414141414, + "eps": 0, + "loss/policy_avg": 0.15740007162094116, + "loss/value_avg": 3.3765735626220703, + "lr": 9.999714542826806e-06, + "objective/entropy": -104.05907440185547, + "objective/kl": 9.103609085083008, + "objective/non_score_reward": -0.45518046617507935, + "objective/rlhf_reward": 9.693256378173828, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0011231796815991402, + "policy/clipfrac_avg": 0.014930792152881622, + "policy/entropy_avg": 0.22347259521484375, + "step": 67, + "val/clipfrac_avg": 0.19888150691986084, + "val/num_eos_tokens": 446, + "val/ratio": 0.9885534048080444, + "val/ratio_var": 0.00011293533316347748 + }, + { + "episode": 1088, + "epoch": 0.05494949494949495, + "eps": 0, + "loss/policy_avg": -0.08715461194515228, + "loss/value_avg": 0.9404190182685852, + "lr": 9.999553975554186e-06, + "objective/entropy": -45.16947555541992, + "objective/kl": 11.139631271362305, + "objective/non_score_reward": -0.5569815635681152, + "objective/rlhf_reward": 7.778955936431885, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0012054555118083954, + "policy/clipfrac_avg": 0.008249208331108093, + "policy/entropy_avg": 0.19194626808166504, + "step": 68, + "val/clipfrac_avg": 0.24800170958042145, + "val/num_eos_tokens": 79, + "val/ratio": 0.9870567917823792, + "val/ratio_var": 9.06415589270182e-05 + }, + { + "episode": 1104, + "epoch": 0.055757575757575756, + "eps": 0, + "loss/policy_avg": -0.24446356296539307, + "loss/value_avg": 2.7453207969665527, + "lr": 9.999357728999657e-06, + "objective/entropy": -115.63008117675781, + "objective/kl": 8.01171588897705, + "objective/non_score_reward": -0.400585800409317, + "objective/rlhf_reward": 10.536914825439453, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0008125576423481107, + "policy/clipfrac_avg": 0.011305256746709347, + "policy/entropy_avg": 0.14436888694763184, + "step": 69, + "val/clipfrac_avg": 0.2002556025981903, + "val/num_eos_tokens": 396, + "val/ratio": 0.9931237697601318, + "val/ratio_var": 0.0001478532503824681 + }, + { + "episode": 1120, + "epoch": 0.05656565656565657, + "eps": 0, + "loss/policy_avg": -0.27069106698036194, + "loss/value_avg": 3.394862413406372, + "lr": 9.999125804563732e-06, + "objective/entropy": -155.4931640625, + "objective/kl": 10.501497268676758, + "objective/non_score_reward": -0.5250749588012695, + "objective/rlhf_reward": 9.31476879119873, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0009476456907577813, + "policy/clipfrac_avg": 0.008611776866018772, + "policy/entropy_avg": 0.17398953437805176, + "step": 70, + "val/clipfrac_avg": 0.258465975522995, + "val/num_eos_tokens": 621, + "val/ratio": 0.9879174828529358, + "val/ratio_var": 0.0001811307593015954 + }, + { + "episode": 1136, + "epoch": 0.057373737373737375, + "eps": 0, + "loss/policy_avg": 0.06734678149223328, + "loss/value_avg": 2.078795909881592, + "lr": 9.99885820390154e-06, + "objective/entropy": -32.39612579345703, + "objective/kl": 11.062005996704102, + "objective/non_score_reward": -0.5531003475189209, + "objective/rlhf_reward": 10.0718994140625, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0010651771444827318, + "policy/clipfrac_avg": 0.018125716596841812, + "policy/entropy_avg": 0.2026987075805664, + "step": 71, + "val/clipfrac_avg": 0.22745996713638306, + "val/num_eos_tokens": 56, + "val/ratio": 0.9876272678375244, + "val/ratio_var": 4.869156327913515e-05 + }, + { + "episode": 1152, + "epoch": 0.05818181818181818, + "eps": 0, + "loss/policy_avg": 0.360561728477478, + "loss/value_avg": 3.147411346435547, + "lr": 9.99855492892281e-06, + "objective/entropy": -84.52821350097656, + "objective/kl": 8.335628509521484, + "objective/non_score_reward": -0.4167814254760742, + "objective/rlhf_reward": 7.708218574523926, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0018768985755741596, + "policy/clipfrac_avg": 0.03232686221599579, + "policy/entropy_avg": 0.30700182914733887, + "step": 72, + "val/clipfrac_avg": 0.21321573853492737, + "val/num_eos_tokens": 447, + "val/ratio": 0.9917864799499512, + "val/ratio_var": 8.937316306401044e-05 + }, + { + "episode": 1168, + "epoch": 0.05898989898989899, + "eps": 0, + "loss/policy_avg": -0.040888212621212006, + "loss/value_avg": 1.4504942893981934, + "lr": 9.998215981791862e-06, + "objective/entropy": -81.507568359375, + "objective/kl": 14.068692207336426, + "objective/non_score_reward": -0.7034346461296082, + "objective/rlhf_reward": 8.046566009521484, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0029386423993855715, + "policy/clipfrac_avg": 0.043007656931877136, + "policy/entropy_avg": 0.19576501846313477, + "step": 73, + "val/clipfrac_avg": 0.07042980194091797, + "val/num_eos_tokens": 149, + "val/ratio": 0.9879980087280273, + "val/ratio_var": 7.574875780846924e-05 + }, + { + "episode": 1184, + "epoch": 0.0597979797979798, + "eps": 0, + "loss/policy_avg": -0.14120109379291534, + "loss/value_avg": 2.754734516143799, + "lr": 9.997841364927587e-06, + "objective/entropy": -95.04641723632812, + "objective/kl": 9.671992301940918, + "objective/non_score_reward": -0.48359963297843933, + "objective/rlhf_reward": 10.774212837219238, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0007605117280036211, + "policy/clipfrac_avg": 0.009797083213925362, + "policy/entropy_avg": 0.148110032081604, + "step": 74, + "val/clipfrac_avg": 0.1554110050201416, + "val/num_eos_tokens": 261, + "val/ratio": 0.9907310009002686, + "val/ratio_var": 6.644838867941871e-05 + }, + { + "episode": 1200, + "epoch": 0.06060606060606061, + "eps": 0, + "loss/policy_avg": 0.45924872159957886, + "loss/value_avg": 2.2186315059661865, + "lr": 9.99743108100344e-06, + "objective/entropy": -133.35256958007812, + "objective/kl": 10.376567840576172, + "objective/non_score_reward": -0.5188283920288086, + "objective/rlhf_reward": 5.967499732971191, + "objective/scores": 6.5, + "policy/approxkl_avg": 0.0010453840950503945, + "policy/clipfrac_avg": 0.027546092867851257, + "policy/entropy_avg": 0.23077678680419922, + "step": 75, + "val/clipfrac_avg": 0.2535858154296875, + "val/num_eos_tokens": 319, + "val/ratio": 0.9903374910354614, + "val/ratio_var": 3.860687866108492e-05 + }, + { + "episode": 1216, + "epoch": 0.061414141414141414, + "eps": 0, + "loss/policy_avg": 0.05463395267724991, + "loss/value_avg": 1.4657877683639526, + "lr": 9.996985132947408e-06, + "objective/entropy": -95.95547485351562, + "objective/kl": 11.160400390625, + "objective/non_score_reward": -0.5580199956893921, + "objective/rlhf_reward": 9.328699111938477, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0008519854163751006, + "policy/clipfrac_avg": 0.014135795645415783, + "policy/entropy_avg": 0.1674652099609375, + "step": 76, + "val/clipfrac_avg": 0.11248327791690826, + "val/num_eos_tokens": 231, + "val/ratio": 0.9904640316963196, + "val/ratio_var": 5.162335583008826e-05 + }, + { + "episode": 1232, + "epoch": 0.06222222222222222, + "eps": 0, + "loss/policy_avg": 0.0632552057504654, + "loss/value_avg": 1.2827647924423218, + "lr": 9.996503523941994e-06, + "objective/entropy": -142.799560546875, + "objective/kl": 8.947230339050293, + "objective/non_score_reward": -0.4473615288734436, + "objective/rlhf_reward": 8.033106803894043, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0007396876462735236, + "policy/clipfrac_avg": 0.006773149594664574, + "policy/entropy_avg": 0.16584384441375732, + "step": 77, + "val/clipfrac_avg": 0.07109327614307404, + "val/num_eos_tokens": 303, + "val/ratio": 0.9911788105964661, + "val/ratio_var": 5.5046573834260926e-05 + }, + { + "episode": 1248, + "epoch": 0.06303030303030303, + "eps": 0, + "loss/policy_avg": -0.7663191556930542, + "loss/value_avg": 2.673025608062744, + "lr": 9.9959862574242e-06, + "objective/entropy": -138.09356689453125, + "objective/kl": 9.970010757446289, + "objective/non_score_reward": -0.4985005855560303, + "objective/rlhf_reward": 10.21243667602539, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.000907164765521884, + "policy/clipfrac_avg": 0.008060376159846783, + "policy/entropy_avg": 0.15853583812713623, + "step": 78, + "val/clipfrac_avg": 0.2591398358345032, + "val/num_eos_tokens": 47, + "val/ratio": 0.9908342361450195, + "val/ratio_var": 5.449764648801647e-05 + }, + { + "episode": 1264, + "epoch": 0.06383838383838383, + "eps": 0, + "loss/policy_avg": 0.30310457944869995, + "loss/value_avg": 2.481609344482422, + "lr": 9.995433337085492e-06, + "objective/entropy": -55.90776062011719, + "objective/kl": 8.261874198913574, + "objective/non_score_reward": -0.4130937457084656, + "objective/rlhf_reward": 8.180656433105469, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.00221771071664989, + "policy/clipfrac_avg": 0.018315991386771202, + "policy/entropy_avg": 0.19953298568725586, + "step": 79, + "val/clipfrac_avg": 0.29067862033843994, + "val/num_eos_tokens": 16, + "val/ratio": 0.987062931060791, + "val/ratio_var": 7.512731099268422e-05 + }, + { + "episode": 1280, + "epoch": 0.06464646464646465, + "eps": 0, + "loss/policy_avg": -0.13517440855503082, + "loss/value_avg": 1.4077324867248535, + "lr": 9.994844766871785e-06, + "objective/entropy": -114.97703552246094, + "objective/kl": 10.453306198120117, + "objective/non_score_reward": -0.5226653218269348, + "objective/rlhf_reward": 8.910928726196289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008272784180007875, + "policy/clipfrac_avg": 0.008443601429462433, + "policy/entropy_avg": 0.1677337884902954, + "step": 80, + "val/clipfrac_avg": 0.24559587240219116, + "val/num_eos_tokens": 597, + "val/ratio": 0.9903725981712341, + "val/ratio_var": 9.728262375574559e-05 + }, + { + "episode": 1296, + "epoch": 0.06545454545454546, + "eps": 0, + "loss/policy_avg": -0.22855637967586517, + "loss/value_avg": 2.5628278255462646, + "lr": 9.994220550983404e-06, + "objective/entropy": -65.29562377929688, + "objective/kl": 8.214365005493164, + "objective/non_score_reward": -0.41071823239326477, + "objective/rlhf_reward": 9.48771858215332, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0007956020417623222, + "policy/clipfrac_avg": 0.006707598455250263, + "policy/entropy_avg": 0.13605308532714844, + "step": 81, + "val/clipfrac_avg": 0.12181060016155243, + "val/num_eos_tokens": 276, + "val/ratio": 0.9903298020362854, + "val/ratio_var": 5.526099630515091e-05 + }, + { + "episode": 1312, + "epoch": 0.06626262626262626, + "eps": 0, + "loss/policy_avg": 0.3731042444705963, + "loss/value_avg": 1.8982326984405518, + "lr": 9.993560693875065e-06, + "objective/entropy": -71.39965057373047, + "objective/kl": 8.709017753601074, + "objective/non_score_reward": -0.43545088171958923, + "objective/rlhf_reward": 7.359470844268799, + "objective/scores": 7.78125, + "policy/approxkl_avg": 0.0009509660303592682, + "policy/clipfrac_avg": 0.01494086068123579, + "policy/entropy_avg": 0.18790435791015625, + "step": 82, + "val/clipfrac_avg": 0.34438806772232056, + "val/num_eos_tokens": 276, + "val/ratio": 0.9874451160430908, + "val/ratio_var": 3.9616661524632946e-05 + }, + { + "episode": 1328, + "epoch": 0.06707070707070707, + "eps": 0, + "loss/policy_avg": -0.06645295768976212, + "loss/value_avg": 1.9543490409851074, + "lr": 9.992865200255829e-06, + "objective/entropy": -186.06890869140625, + "objective/kl": 9.877692222595215, + "objective/non_score_reward": -0.4938845634460449, + "objective/rlhf_reward": 7.853771686553955, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008737960597500205, + "policy/clipfrac_avg": 0.030746009200811386, + "policy/entropy_avg": 0.27783632278442383, + "step": 83, + "val/clipfrac_avg": 0.21427029371261597, + "val/num_eos_tokens": 434, + "val/ratio": 0.9895691871643066, + "val/ratio_var": 0.0001006032107397914 + }, + { + "episode": 1344, + "epoch": 0.06787878787878789, + "eps": 0, + "loss/policy_avg": 0.06399939954280853, + "loss/value_avg": 1.2927840948104858, + "lr": 9.992134075089085e-06, + "objective/entropy": -112.0244140625, + "objective/kl": 10.45096206665039, + "objective/non_score_reward": -0.5225480794906616, + "objective/rlhf_reward": 8.91495132446289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010605708230286837, + "policy/clipfrac_avg": 0.01699572056531906, + "policy/entropy_avg": 0.15537548065185547, + "step": 84, + "val/clipfrac_avg": 0.06052979826927185, + "val/num_eos_tokens": 693, + "val/ratio": 0.9891489744186401, + "val/ratio_var": 7.55400033085607e-05 + }, + { + "episode": 1360, + "epoch": 0.06868686868686869, + "eps": 0, + "loss/policy_avg": 0.05262535810470581, + "loss/value_avg": 1.9076130390167236, + "lr": 9.991367323592499e-06, + "objective/entropy": -110.17214965820312, + "objective/kl": 7.841750621795654, + "objective/non_score_reward": -0.39208751916885376, + "objective/rlhf_reward": 9.154787063598633, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0011510876938700676, + "policy/clipfrac_avg": 0.029715364798903465, + "policy/entropy_avg": 0.14819884300231934, + "step": 85, + "val/clipfrac_avg": 0.13365106284618378, + "val/num_eos_tokens": 172, + "val/ratio": 0.9908526539802551, + "val/ratio_var": 6.175693852128461e-05 + }, + { + "episode": 1376, + "epoch": 0.0694949494949495, + "eps": 0, + "loss/policy_avg": 0.29705333709716797, + "loss/value_avg": 2.059380531311035, + "lr": 9.99056495123799e-06, + "objective/entropy": -125.11438751220703, + "objective/kl": 10.169289588928223, + "objective/non_score_reward": -0.5084644556045532, + "objective/rlhf_reward": 7.858722686767578, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008927697199396789, + "policy/clipfrac_avg": 0.014697541482746601, + "policy/entropy_avg": 0.17176449298858643, + "step": 86, + "val/clipfrac_avg": 0.1657203733921051, + "val/num_eos_tokens": 262, + "val/ratio": 0.9888587594032288, + "val/ratio_var": 8.45455433591269e-05 + }, + { + "episode": 1392, + "epoch": 0.0703030303030303, + "eps": 0, + "loss/policy_avg": -0.310147762298584, + "loss/value_avg": 1.6057798862457275, + "lr": 9.989726963751683e-06, + "objective/entropy": -110.98075866699219, + "objective/kl": 9.372626304626465, + "objective/non_score_reward": -0.4686313271522522, + "objective/rlhf_reward": 9.06261920928955, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009293662733398378, + "policy/clipfrac_avg": 0.013371942564845085, + "policy/entropy_avg": 0.2098407745361328, + "step": 87, + "val/clipfrac_avg": 0.14167499542236328, + "val/num_eos_tokens": 219, + "val/ratio": 0.9893955588340759, + "val/ratio_var": 0.00011555494711501524 + }, + { + "episode": 1408, + "epoch": 0.07111111111111111, + "eps": 0, + "loss/policy_avg": -0.36583343148231506, + "loss/value_avg": 1.4973384141921997, + "lr": 9.98885336711387e-06, + "objective/entropy": -119.2513198852539, + "objective/kl": 8.599248886108398, + "objective/non_score_reward": -0.4299624264240265, + "objective/rlhf_reward": 9.562225341796875, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0008632695535197854, + "policy/clipfrac_avg": 0.014250310137867928, + "policy/entropy_avg": 0.1634058952331543, + "step": 88, + "val/clipfrac_avg": 0.19998914003372192, + "val/num_eos_tokens": 380, + "val/ratio": 0.9901472330093384, + "val/ratio_var": 9.213508747052401e-05 + }, + { + "episode": 1424, + "epoch": 0.07191919191919192, + "eps": 0, + "loss/policy_avg": 0.3967055678367615, + "loss/value_avg": 2.1769919395446777, + "lr": 9.987944167558963e-06, + "objective/entropy": -137.1684112548828, + "objective/kl": 7.493285655975342, + "objective/non_score_reward": -0.374664306640625, + "objective/rlhf_reward": 6.914398193359375, + "objective/scores": 7.28125, + "policy/approxkl_avg": 0.0010005123913288116, + "policy/clipfrac_avg": 0.012202544137835503, + "policy/entropy_avg": 0.13602828979492188, + "step": 89, + "val/clipfrac_avg": 0.22100284695625305, + "val/num_eos_tokens": 391, + "val/ratio": 0.993422269821167, + "val/ratio_var": 0.0001790324313333258 + }, + { + "episode": 1440, + "epoch": 0.07272727272727272, + "eps": 0, + "loss/policy_avg": -0.166255384683609, + "loss/value_avg": 0.6386685967445374, + "lr": 9.986999371575465e-06, + "objective/entropy": -102.01996612548828, + "objective/kl": 9.35251235961914, + "objective/non_score_reward": -0.46762561798095703, + "objective/rlhf_reward": 8.868311882019043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007846959633752704, + "policy/clipfrac_avg": 0.007588379550725222, + "policy/entropy_avg": 0.18400812149047852, + "step": 90, + "val/clipfrac_avg": 0.26963502168655396, + "val/num_eos_tokens": 381, + "val/ratio": 0.9904755353927612, + "val/ratio_var": 4.9346734158461913e-05 + }, + { + "episode": 1456, + "epoch": 0.07353535353535354, + "eps": 0, + "loss/policy_avg": 0.06461673974990845, + "loss/value_avg": 1.8350507020950317, + "lr": 9.986018985905901e-06, + "objective/entropy": -52.68136215209961, + "objective/kl": 6.7079644203186035, + "objective/non_score_reward": -0.33539825677871704, + "objective/rlhf_reward": 10.49272632598877, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0012186269741505384, + "policy/clipfrac_avg": 0.006110272370278835, + "policy/entropy_avg": 0.21361064910888672, + "step": 91, + "val/clipfrac_avg": 0.15903371572494507, + "val/num_eos_tokens": 142, + "val/ratio": 0.9849614500999451, + "val/ratio_var": 0.00012401219282764941 + }, + { + "episode": 1472, + "epoch": 0.07434343434343435, + "eps": 0, + "loss/policy_avg": 0.2874661684036255, + "loss/value_avg": 1.235567569732666, + "lr": 9.985003017546789e-06, + "objective/entropy": -110.07864379882812, + "objective/kl": 11.082052230834961, + "objective/non_score_reward": -0.5541026592254639, + "objective/rlhf_reward": 8.570897102355957, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0008647169452160597, + "policy/clipfrac_avg": 0.01752229407429695, + "policy/entropy_avg": 0.1840806007385254, + "step": 92, + "val/clipfrac_avg": 0.11287079006433487, + "val/num_eos_tokens": 48, + "val/ratio": 0.9882228374481201, + "val/ratio_var": 6.303517875494435e-05 + }, + { + "episode": 1488, + "epoch": 0.07515151515151515, + "eps": 0, + "loss/policy_avg": -1.0834553241729736, + "loss/value_avg": 2.374274969100952, + "lr": 9.983951473748579e-06, + "objective/entropy": -110.14735412597656, + "objective/kl": 10.317655563354492, + "objective/non_score_reward": -0.5158827900886536, + "objective/rlhf_reward": 10.44505500793457, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007985912961885333, + "policy/clipfrac_avg": 0.0036714496091008186, + "policy/entropy_avg": 0.1591578722000122, + "step": 93, + "val/clipfrac_avg": 0.26723405718803406, + "val/num_eos_tokens": 348, + "val/ratio": 0.989249587059021, + "val/ratio_var": 6.81774690747261e-05 + }, + { + "episode": 1504, + "epoch": 0.07595959595959596, + "eps": 0, + "loss/policy_avg": -0.016785837709903717, + "loss/value_avg": 2.52266788482666, + "lr": 9.982864362015606e-06, + "objective/entropy": -26.387767791748047, + "objective/kl": 11.676578521728516, + "objective/non_score_reward": -0.5838289260864258, + "objective/rlhf_reward": 8.232576370239258, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.00300201796926558, + "policy/clipfrac_avg": 0.014051968231797218, + "policy/entropy_avg": 0.23279857635498047, + "step": 94, + "val/clipfrac_avg": 0.19883441925048828, + "val/num_eos_tokens": 74, + "val/ratio": 0.9866381883621216, + "val/ratio_var": 6.660662620561197e-05 + }, + { + "episode": 1520, + "epoch": 0.07676767676767676, + "eps": 0, + "loss/policy_avg": -0.1366100311279297, + "loss/value_avg": 2.2388007640838623, + "lr": 9.981741690106035e-06, + "objective/entropy": -111.26895141601562, + "objective/kl": 11.7849760055542, + "objective/non_score_reward": -0.589248776435852, + "objective/rlhf_reward": 9.957626342773438, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0032595903612673283, + "policy/clipfrac_avg": 0.025392604991793633, + "policy/entropy_avg": 0.23312854766845703, + "step": 95, + "val/clipfrac_avg": 0.1502402424812317, + "val/num_eos_tokens": 15, + "val/ratio": 0.9824517965316772, + "val/ratio_var": 0.0003387684701010585 + }, + { + "episode": 1536, + "epoch": 0.07757575757575758, + "eps": 0, + "loss/policy_avg": 0.31284308433532715, + "loss/value_avg": 3.7777535915374756, + "lr": 9.980583466031808e-06, + "objective/entropy": -67.97806549072266, + "objective/kl": 9.119251251220703, + "objective/non_score_reward": -0.45596253871917725, + "objective/rlhf_reward": 7.071381092071533, + "objective/scores": 7.53125, + "policy/approxkl_avg": 0.0009370450279675424, + "policy/clipfrac_avg": 0.023228613659739494, + "policy/entropy_avg": 0.19547653198242188, + "step": 96, + "val/clipfrac_avg": 0.30271416902542114, + "val/num_eos_tokens": 180, + "val/ratio": 0.9877786636352539, + "val/ratio_var": 7.666354213142768e-05 + }, + { + "episode": 1552, + "epoch": 0.07838383838383839, + "eps": 0, + "loss/policy_avg": -0.260562539100647, + "loss/value_avg": 1.9976837635040283, + "lr": 9.979389698058581e-06, + "objective/entropy": -168.7757568359375, + "objective/kl": 10.928716659545898, + "objective/non_score_reward": -0.5464357733726501, + "objective/rlhf_reward": 7.793407917022705, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0010802671313285828, + "policy/clipfrac_avg": 0.013121331110596657, + "policy/entropy_avg": 0.14757728576660156, + "step": 97, + "val/clipfrac_avg": 0.17170178890228271, + "val/num_eos_tokens": 530, + "val/ratio": 0.992108166217804, + "val/ratio_var": 0.00020361572387628257 + }, + { + "episode": 1568, + "epoch": 0.07919191919191919, + "eps": 0, + "loss/policy_avg": 0.20252731442451477, + "loss/value_avg": 1.8869277238845825, + "lr": 9.978160394705669e-06, + "objective/entropy": -88.97151184082031, + "objective/kl": 8.581636428833008, + "objective/non_score_reward": -0.42908182740211487, + "objective/rlhf_reward": 7.738887310028076, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008207992068491876, + "policy/clipfrac_avg": 0.014891392551362514, + "policy/entropy_avg": 0.24853515625, + "step": 98, + "val/clipfrac_avg": 0.14818435907363892, + "val/num_eos_tokens": 287, + "val/ratio": 0.9901211261749268, + "val/ratio_var": 6.446534825954586e-05 + }, + { + "episode": 1584, + "epoch": 0.08, + "eps": 0, + "loss/policy_avg": 0.4129256010055542, + "loss/value_avg": 1.6777870655059814, + "lr": 9.976895564745993e-06, + "objective/entropy": -70.99946594238281, + "objective/kl": 10.816564559936523, + "objective/non_score_reward": -0.540828287601471, + "objective/rlhf_reward": 8.123233795166016, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002382381819188595, + "policy/clipfrac_avg": 0.03098738193511963, + "policy/entropy_avg": 0.19532859325408936, + "step": 99, + "val/clipfrac_avg": 0.17884066700935364, + "val/num_eos_tokens": 287, + "val/ratio": 0.9863871932029724, + "val/ratio_var": 0.00017909934103954583 + }, + { + "episode": 1600, + "epoch": 0.08080808080808081, + "eps": 0, + "loss/policy_avg": 0.07570860534906387, + "loss/value_avg": 0.5786488652229309, + "lr": 9.975595217205995e-06, + "objective/entropy": -94.05448913574219, + "objective/kl": 10.959748268127441, + "objective/non_score_reward": -0.5479874610900879, + "objective/rlhf_reward": 9.295762062072754, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0010338419815525413, + "policy/clipfrac_avg": 0.009063776582479477, + "policy/entropy_avg": 0.19421935081481934, + "step": 100, + "val/clipfrac_avg": 0.13884295523166656, + "val/num_eos_tokens": 14, + "val/ratio": 0.9873470664024353, + "val/ratio_var": 7.10823223926127e-05 + }, + { + "episode": 1616, + "epoch": 0.08161616161616161, + "eps": 0, + "loss/policy_avg": 0.18178647756576538, + "loss/value_avg": 3.246936559677124, + "lr": 9.974259361365604e-06, + "objective/entropy": -147.2283172607422, + "objective/kl": 9.482419967651367, + "objective/non_score_reward": -0.4741210341453552, + "objective/rlhf_reward": 9.45361328125, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.000866247049998492, + "policy/clipfrac_avg": 0.011027923785150051, + "policy/entropy_avg": 0.18662023544311523, + "step": 101, + "val/clipfrac_avg": 0.21203863620758057, + "val/num_eos_tokens": 62, + "val/ratio": 0.989406406879425, + "val/ratio_var": 9.00850209291093e-05 + }, + { + "episode": 1632, + "epoch": 0.08242424242424243, + "eps": 0, + "loss/policy_avg": -0.3622581660747528, + "loss/value_avg": 3.517216205596924, + "lr": 9.97288800675814e-06, + "objective/entropy": -102.03694915771484, + "objective/kl": 11.228017807006836, + "objective/non_score_reward": -0.5614008903503418, + "objective/rlhf_reward": 9.1944580078125, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009244791581295431, + "policy/clipfrac_avg": 0.02670031040906906, + "policy/entropy_avg": 0.19944334030151367, + "step": 102, + "val/clipfrac_avg": 0.22908943891525269, + "val/num_eos_tokens": 360, + "val/ratio": 0.9883830547332764, + "val/ratio_var": 8.266136137535796e-05 + }, + { + "episode": 1648, + "epoch": 0.08323232323232323, + "eps": 0, + "loss/policy_avg": 0.004343941807746887, + "loss/value_avg": 3.0538196563720703, + "lr": 9.97148116317027e-06, + "objective/entropy": -142.39923095703125, + "objective/kl": 11.77578067779541, + "objective/non_score_reward": -0.5887890458106995, + "objective/rlhf_reward": 9.028398513793945, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000976751558482647, + "policy/clipfrac_avg": 0.013522524386644363, + "policy/entropy_avg": 0.26236867904663086, + "step": 103, + "val/clipfrac_avg": 0.21960067749023438, + "val/num_eos_tokens": 9, + "val/ratio": 0.9883174896240234, + "val/ratio_var": 0.00010564823605818674 + }, + { + "episode": 1664, + "epoch": 0.08404040404040404, + "eps": 0, + "loss/policy_avg": 0.4931744933128357, + "loss/value_avg": 2.0421762466430664, + "lr": 9.970038840641918e-06, + "objective/entropy": -71.44817352294922, + "objective/kl": 12.531501770019531, + "objective/non_score_reward": -0.6265751123428345, + "objective/rlhf_reward": 8.525768280029297, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0010512301232665777, + "policy/clipfrac_avg": 0.03077285923063755, + "policy/entropy_avg": 0.20035934448242188, + "step": 104, + "val/clipfrac_avg": 0.23763741552829742, + "val/num_eos_tokens": 23, + "val/ratio": 0.9862467050552368, + "val/ratio_var": 0.00010008800018113106 + }, + { + "episode": 1680, + "epoch": 0.08484848484848485, + "eps": 0, + "loss/policy_avg": 0.15791772305965424, + "loss/value_avg": 1.9950876235961914, + "lr": 9.968561049466214e-06, + "objective/entropy": -60.47146224975586, + "objective/kl": 11.113540649414062, + "objective/non_score_reward": -0.555677056312561, + "objective/rlhf_reward": 8.44432258605957, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001011199550703168, + "policy/clipfrac_avg": 0.016472183167934418, + "policy/entropy_avg": 0.15747523307800293, + "step": 105, + "val/clipfrac_avg": 0.2771484851837158, + "val/num_eos_tokens": 26, + "val/ratio": 0.9897866249084473, + "val/ratio_var": 4.6754288632655516e-05 + }, + { + "episode": 1696, + "epoch": 0.08565656565656565, + "eps": 0, + "loss/policy_avg": 0.14141756296157837, + "loss/value_avg": 1.6132752895355225, + "lr": 9.967047800189401e-06, + "objective/entropy": -109.22718811035156, + "objective/kl": 12.041143417358398, + "objective/non_score_reward": -0.6020572185516357, + "objective/rlhf_reward": 9.976067543029785, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.001960658235475421, + "policy/clipfrac_avg": 0.02854878082871437, + "policy/entropy_avg": 0.14309561252593994, + "step": 106, + "val/clipfrac_avg": 0.16923227906227112, + "val/num_eos_tokens": 19, + "val/ratio": 0.9895997643470764, + "val/ratio_var": 5.035091453464702e-05 + }, + { + "episode": 1712, + "epoch": 0.08646464646464647, + "eps": 0, + "loss/policy_avg": -0.13957123458385468, + "loss/value_avg": 0.8985839486122131, + "lr": 9.965499103610775e-06, + "objective/entropy": -166.47471618652344, + "objective/kl": 11.278413772583008, + "objective/non_score_reward": -0.5639207363128662, + "objective/rlhf_reward": 11.272016525268555, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0008130659116432071, + "policy/clipfrac_avg": 0.009939244017004967, + "policy/entropy_avg": 0.13879013061523438, + "step": 107, + "val/clipfrac_avg": 0.10860806703567505, + "val/num_eos_tokens": 232, + "val/ratio": 0.9910627603530884, + "val/ratio_var": 4.761793388752267e-05 + }, + { + "episode": 1728, + "epoch": 0.08727272727272728, + "eps": 0, + "loss/policy_avg": -0.11392714083194733, + "loss/value_avg": 1.136710524559021, + "lr": 9.963914970782594e-06, + "objective/entropy": -81.45556640625, + "objective/kl": 10.213882446289062, + "objective/non_score_reward": -0.510694146156311, + "objective/rlhf_reward": 9.04399299621582, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0012008333578705788, + "policy/clipfrac_avg": 0.010314267128705978, + "policy/entropy_avg": 0.22310292720794678, + "step": 108, + "val/clipfrac_avg": 0.16719487309455872, + "val/num_eos_tokens": 724, + "val/ratio": 0.9873642921447754, + "val/ratio_var": 0.000118928866868373 + }, + { + "episode": 1744, + "epoch": 0.08808080808080808, + "eps": 0, + "loss/policy_avg": 0.02373083308339119, + "loss/value_avg": 0.7551393508911133, + "lr": 9.962295413010013e-06, + "objective/entropy": -23.411705017089844, + "objective/kl": 17.010793685913086, + "objective/non_score_reward": -0.8505396842956543, + "objective/rlhf_reward": 6.887741565704346, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0017146767349913716, + "policy/clipfrac_avg": 0.007722174748778343, + "policy/entropy_avg": 0.24081039428710938, + "step": 109, + "val/clipfrac_avg": 0.265152245759964, + "val/num_eos_tokens": 92, + "val/ratio": 0.9839531779289246, + "val/ratio_var": 0.00013079243944957852 + }, + { + "episode": 1760, + "epoch": 0.08888888888888889, + "eps": 0, + "loss/policy_avg": -0.029323376715183258, + "loss/value_avg": 0.6305721998214722, + "lr": 9.960640441850997e-06, + "objective/entropy": -49.22315216064453, + "objective/kl": 17.350746154785156, + "objective/non_score_reward": -0.867537260055542, + "objective/rlhf_reward": 7.573868751525879, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.001081926515325904, + "policy/clipfrac_avg": 0.012329247780144215, + "policy/entropy_avg": 0.19385910034179688, + "step": 110, + "val/clipfrac_avg": 0.14187291264533997, + "val/num_eos_tokens": 469, + "val/ratio": 0.9876461625099182, + "val/ratio_var": 5.4690557590220124e-05 + }, + { + "episode": 1776, + "epoch": 0.08969696969696969, + "eps": 0, + "loss/policy_avg": -0.09420577436685562, + "loss/value_avg": 1.1037640571594238, + "lr": 9.95895006911623e-06, + "objective/entropy": -99.91004180908203, + "objective/kl": 13.64469051361084, + "objective/non_score_reward": -0.682234525680542, + "objective/rlhf_reward": 9.591202735900879, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0013218795647844672, + "policy/clipfrac_avg": 0.023597117513418198, + "policy/entropy_avg": 0.19385337829589844, + "step": 111, + "val/clipfrac_avg": 0.16007894277572632, + "val/num_eos_tokens": 23, + "val/ratio": 0.9876752495765686, + "val/ratio_var": 9.703996329335496e-05 + }, + { + "episode": 1792, + "epoch": 0.0905050505050505, + "eps": 0, + "loss/policy_avg": -0.012303899973630905, + "loss/value_avg": 1.2762086391448975, + "lr": 9.957224306869053e-06, + "objective/entropy": -43.240875244140625, + "objective/kl": 15.633418083190918, + "objective/non_score_reward": -0.7816709280014038, + "objective/rlhf_reward": 7.913640975952148, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.002886019414290786, + "policy/clipfrac_avg": 0.012701047584414482, + "policy/entropy_avg": 0.23091506958007812, + "step": 112, + "val/clipfrac_avg": 0.23217324912548065, + "val/num_eos_tokens": 18, + "val/ratio": 0.9861587285995483, + "val/ratio_var": 0.0003747409500647336 + }, + { + "episode": 1808, + "epoch": 0.09131313131313132, + "eps": 0, + "loss/policy_avg": -0.5104568004608154, + "loss/value_avg": 2.1639163494110107, + "lr": 9.95546316742535e-06, + "objective/entropy": -69.88031768798828, + "objective/kl": 12.853160858154297, + "objective/non_score_reward": -0.6426580548286438, + "objective/rlhf_reward": 9.568279266357422, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011324469232931733, + "policy/clipfrac_avg": 0.004497843328863382, + "policy/entropy_avg": 0.16321039199829102, + "step": 113, + "val/clipfrac_avg": 0.28476184606552124, + "val/num_eos_tokens": 59, + "val/ratio": 0.9893928170204163, + "val/ratio_var": 6.093212141422555e-05 + }, + { + "episode": 1824, + "epoch": 0.09212121212121212, + "eps": 0, + "loss/policy_avg": -0.5700721740722656, + "loss/value_avg": 3.390981674194336, + "lr": 9.953666663353485e-06, + "objective/entropy": -92.63809204101562, + "objective/kl": 13.041144371032715, + "objective/non_score_reward": -0.6520571708679199, + "objective/rlhf_reward": 10.613567352294922, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0009989608079195023, + "policy/clipfrac_avg": 0.012498335912823677, + "policy/entropy_avg": 0.1557478904724121, + "step": 114, + "val/clipfrac_avg": 0.16689905524253845, + "val/num_eos_tokens": 337, + "val/ratio": 0.9884234666824341, + "val/ratio_var": 0.0001190627517644316 + }, + { + "episode": 1840, + "epoch": 0.09292929292929293, + "eps": 0, + "loss/policy_avg": -0.037001632153987885, + "loss/value_avg": 1.2843832969665527, + "lr": 9.951834807474191e-06, + "objective/entropy": -62.3432502746582, + "objective/kl": 11.188087463378906, + "objective/non_score_reward": -0.5594043731689453, + "objective/rlhf_reward": 9.198408126831055, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008078927057795227, + "policy/clipfrac_avg": 0.00809546373784542, + "policy/entropy_avg": 0.16070127487182617, + "step": 115, + "val/clipfrac_avg": 0.14679771661758423, + "val/num_eos_tokens": 491, + "val/ratio": 0.9907799363136292, + "val/ratio_var": 6.536444561788812e-05 + }, + { + "episode": 1856, + "epoch": 0.09373737373737374, + "eps": 0, + "loss/policy_avg": 0.392054945230484, + "loss/value_avg": 4.038256645202637, + "lr": 9.9499676128605e-06, + "objective/entropy": -67.57776641845703, + "objective/kl": 13.151895523071289, + "objective/non_score_reward": -0.6575947999954224, + "objective/rlhf_reward": 7.943967819213867, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.001524624414741993, + "policy/clipfrac_avg": 0.03400165215134621, + "policy/entropy_avg": 0.1723041534423828, + "step": 116, + "val/clipfrac_avg": 0.24824200570583344, + "val/num_eos_tokens": 297, + "val/ratio": 0.989495038986206, + "val/ratio_var": 9.145861986326054e-05 + }, + { + "episode": 1872, + "epoch": 0.09454545454545454, + "eps": 0, + "loss/policy_avg": 0.13758663833141327, + "loss/value_avg": 1.6034759283065796, + "lr": 9.948065092837631e-06, + "objective/entropy": -68.10916900634766, + "objective/kl": 8.475647926330566, + "objective/non_score_reward": -0.4237824082374573, + "objective/rlhf_reward": 7.5176239013671875, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007898050826042891, + "policy/clipfrac_avg": 0.006391381844878197, + "policy/entropy_avg": 0.14587292075157166, + "step": 117, + "val/clipfrac_avg": 0.2929156422615051, + "val/num_eos_tokens": 130, + "val/ratio": 0.9905034303665161, + "val/ratio_var": 3.43524880008772e-05 + }, + { + "episode": 1888, + "epoch": 0.09535353535353536, + "eps": 0, + "loss/policy_avg": -0.010858863592147827, + "loss/value_avg": 0.8126935958862305, + "lr": 9.946127260982903e-06, + "objective/entropy": -64.47953033447266, + "objective/kl": 13.444223403930664, + "objective/non_score_reward": -0.6722111701965332, + "objective/rlhf_reward": 7.370757579803467, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0009727979195304215, + "policy/clipfrac_avg": 0.023482225835323334, + "policy/entropy_avg": 0.17380249500274658, + "step": 118, + "val/clipfrac_avg": 0.15926581621170044, + "val/num_eos_tokens": 899, + "val/ratio": 0.9887762069702148, + "val/ratio_var": 4.845508374273777e-05 + }, + { + "episode": 1904, + "epoch": 0.09616161616161616, + "eps": 0, + "loss/policy_avg": 0.05183769389986992, + "loss/value_avg": 0.7353260517120361, + "lr": 9.944154131125643e-06, + "objective/entropy": -142.527587890625, + "objective/kl": 14.874452590942383, + "objective/non_score_reward": -0.7437226176261902, + "objective/rlhf_reward": 7.916433811187744, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.00139252794906497, + "policy/clipfrac_avg": 0.03409278765320778, + "policy/entropy_avg": 0.21674728393554688, + "step": 119, + "val/clipfrac_avg": 0.14925608038902283, + "val/num_eos_tokens": 683, + "val/ratio": 0.9891353845596313, + "val/ratio_var": 7.120384543668479e-05 + }, + { + "episode": 1920, + "epoch": 0.09696969696969697, + "eps": 0, + "loss/policy_avg": 0.15797683596611023, + "loss/value_avg": 2.7395377159118652, + "lr": 9.942145717347077e-06, + "objective/entropy": -30.89788246154785, + "objective/kl": 10.851125717163086, + "objective/non_score_reward": -0.5425562262535095, + "objective/rlhf_reward": 7.843674182891846, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0013285963796079159, + "policy/clipfrac_avg": 0.021273121237754822, + "policy/entropy_avg": 0.20377635955810547, + "step": 120, + "val/clipfrac_avg": 0.16737978160381317, + "val/num_eos_tokens": 345, + "val/ratio": 0.9880788326263428, + "val/ratio_var": 5.1685652579180896e-05 + }, + { + "episode": 1936, + "epoch": 0.09777777777777778, + "eps": 0, + "loss/policy_avg": 0.22806939482688904, + "loss/value_avg": 2.743255615234375, + "lr": 9.940102033980236e-06, + "objective/entropy": -218.7469024658203, + "objective/kl": 14.510229110717773, + "objective/non_score_reward": -0.7255114316940308, + "objective/rlhf_reward": 8.21198844909668, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.001183751504868269, + "policy/clipfrac_avg": 0.045801788568496704, + "policy/entropy_avg": 0.18555426597595215, + "step": 121, + "val/clipfrac_avg": 0.14798504114151, + "val/num_eos_tokens": 350, + "val/ratio": 0.9906567931175232, + "val/ratio_var": 7.052164437482134e-05 + }, + { + "episode": 1952, + "epoch": 0.09858585858585858, + "eps": 0, + "loss/policy_avg": -0.28502777218818665, + "loss/value_avg": 1.3561985492706299, + "lr": 9.93802309560985e-06, + "objective/entropy": -61.34413528442383, + "objective/kl": 10.68556022644043, + "objective/non_score_reward": -0.5342780351638794, + "objective/rlhf_reward": 9.78994083404541, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0008475382928736508, + "policy/clipfrac_avg": 0.007006281986832619, + "policy/entropy_avg": 0.19134902954101562, + "step": 122, + "val/clipfrac_avg": 0.125126451253891, + "val/num_eos_tokens": 19, + "val/ratio": 0.9898977279663086, + "val/ratio_var": 6.245759141165763e-05 + }, + { + "episode": 1968, + "epoch": 0.0993939393939394, + "eps": 0, + "loss/policy_avg": -0.05578336864709854, + "loss/value_avg": 3.2866952419281006, + "lr": 9.935908917072253e-06, + "objective/entropy": -31.413511276245117, + "objective/kl": 11.469106674194336, + "objective/non_score_reward": -0.5734553337097168, + "objective/rlhf_reward": 10.352326393127441, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0017750051338225603, + "policy/clipfrac_avg": 0.019242165610194206, + "policy/entropy_avg": 0.23152542114257812, + "step": 123, + "val/clipfrac_avg": 0.1343041956424713, + "val/num_eos_tokens": 111, + "val/ratio": 0.9851062297821045, + "val/ratio_var": 0.00021594094869215041 + }, + { + "episode": 1984, + "epoch": 0.10020202020202021, + "eps": 0, + "loss/policy_avg": 0.07291892915964127, + "loss/value_avg": 2.094268321990967, + "lr": 9.93375951345526e-06, + "objective/entropy": -41.85675811767578, + "objective/kl": 11.678960800170898, + "objective/non_score_reward": -0.583948016166687, + "objective/rlhf_reward": 8.416051864624023, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.001134565332904458, + "policy/clipfrac_avg": 0.009194762445986271, + "policy/entropy_avg": 0.22137069702148438, + "step": 124, + "val/clipfrac_avg": 0.20579089224338531, + "val/num_eos_tokens": 25, + "val/ratio": 0.9855261445045471, + "val/ratio_var": 0.00015781399270053953 + }, + { + "episode": 2000, + "epoch": 0.10101010101010101, + "eps": 0, + "loss/policy_avg": -0.20081275701522827, + "loss/value_avg": 0.9789823293685913, + "lr": 9.931574900098076e-06, + "objective/entropy": -38.61683654785156, + "objective/kl": 12.147377014160156, + "objective/non_score_reward": -0.6073689460754395, + "objective/rlhf_reward": 8.478569030761719, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0010453371796756983, + "policy/clipfrac_avg": 0.02015816420316696, + "policy/entropy_avg": 0.20670151710510254, + "step": 125, + "val/clipfrac_avg": 0.1644173264503479, + "val/num_eos_tokens": 31, + "val/ratio": 0.9876101016998291, + "val/ratio_var": 5.8746740251081064e-05 + }, + { + "episode": 2016, + "epoch": 0.10181818181818182, + "eps": 0, + "loss/policy_avg": -0.16343963146209717, + "loss/value_avg": 0.9192260503768921, + "lr": 9.92935509259118e-06, + "objective/entropy": -137.78683471679688, + "objective/kl": 14.017477035522461, + "objective/non_score_reward": -0.700873851776123, + "objective/rlhf_reward": 8.428031921386719, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0017388606211170554, + "policy/clipfrac_avg": 0.022260189056396484, + "policy/entropy_avg": 0.17416727542877197, + "step": 126, + "val/clipfrac_avg": 0.23833248019218445, + "val/num_eos_tokens": 155, + "val/ratio": 0.9880942106246948, + "val/ratio_var": 0.00013718391710426658 + }, + { + "episode": 2032, + "epoch": 0.10262626262626262, + "eps": 0, + "loss/policy_avg": -0.10418622195720673, + "loss/value_avg": 2.174149751663208, + "lr": 9.927100106776213e-06, + "objective/entropy": -77.51470947265625, + "objective/kl": 10.212955474853516, + "objective/non_score_reward": -0.5106477737426758, + "objective/rlhf_reward": 8.598727226257324, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0012595304287970066, + "policy/clipfrac_avg": 0.016321320086717606, + "policy/entropy_avg": 0.16196727752685547, + "step": 127, + "val/clipfrac_avg": 0.32252413034439087, + "val/num_eos_tokens": 544, + "val/ratio": 0.992402195930481, + "val/ratio_var": 0.0001610841864021495 + }, + { + "episode": 2048, + "epoch": 0.10343434343434343, + "eps": 0, + "loss/policy_avg": 0.5139014720916748, + "loss/value_avg": 1.2218286991119385, + "lr": 9.924809958745866e-06, + "objective/entropy": -71.08393859863281, + "objective/kl": 9.301607131958008, + "objective/non_score_reward": -0.4650803208351135, + "objective/rlhf_reward": 8.554450988769531, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0004891916178166866, + "policy/clipfrac_avg": 0.006563554517924786, + "policy/entropy_avg": 0.09600615501403809, + "step": 128, + "val/clipfrac_avg": 0.18050439655780792, + "val/num_eos_tokens": 269, + "val/ratio": 0.9929656386375427, + "val/ratio_var": 3.0018733014003374e-05 + }, + { + "episode": 2064, + "epoch": 0.10424242424242425, + "eps": 0, + "loss/policy_avg": -0.4225575625896454, + "loss/value_avg": 0.8013542294502258, + "lr": 9.922484664843763e-06, + "objective/entropy": -131.27288818359375, + "objective/kl": 10.98521900177002, + "objective/non_score_reward": -0.5492609739303589, + "objective/rlhf_reward": 9.403863906860352, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0006737498915754259, + "policy/clipfrac_avg": 0.010425791144371033, + "policy/entropy_avg": 0.12466853857040405, + "step": 129, + "val/clipfrac_avg": 0.3558182716369629, + "val/num_eos_tokens": 268, + "val/ratio": 0.9916852712631226, + "val/ratio_var": 5.279077231534757e-05 + }, + { + "episode": 2080, + "epoch": 0.10505050505050505, + "eps": 0, + "loss/policy_avg": 0.6694027185440063, + "loss/value_avg": 2.389725685119629, + "lr": 9.920124241664349e-06, + "objective/entropy": -134.74630737304688, + "objective/kl": 10.488237380981445, + "objective/non_score_reward": -0.5244117975234985, + "objective/rlhf_reward": 7.772463798522949, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0009083844488486648, + "policy/clipfrac_avg": 0.022758150473237038, + "policy/entropy_avg": 0.17615556716918945, + "step": 130, + "val/clipfrac_avg": 0.3100450038909912, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912290573120117, + "val/ratio_var": 4.8142654122784734e-05 + }, + { + "episode": 2096, + "epoch": 0.10585858585858586, + "eps": 0, + "loss/policy_avg": -0.26812419295310974, + "loss/value_avg": 1.8501536846160889, + "lr": 9.917728706052765e-06, + "objective/entropy": -104.74658966064453, + "objective/kl": 8.395157814025879, + "objective/non_score_reward": -0.4197579026222229, + "objective/rlhf_reward": 8.465007781982422, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0016920489724725485, + "policy/clipfrac_avg": 0.02493302896618843, + "policy/entropy_avg": 0.1534414291381836, + "step": 131, + "val/clipfrac_avg": 0.21037200093269348, + "val/num_eos_tokens": 339, + "val/ratio": 0.990479588508606, + "val/ratio_var": 0.00021930081129539758 + }, + { + "episode": 2112, + "epoch": 0.10666666666666667, + "eps": 0, + "loss/policy_avg": -0.34324708580970764, + "loss/value_avg": 0.6897234916687012, + "lr": 9.915298075104735e-06, + "objective/entropy": -64.35730743408203, + "objective/kl": 13.195899963378906, + "objective/non_score_reward": -0.6597949266433716, + "objective/rlhf_reward": 8.508173942565918, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009221321670338511, + "policy/clipfrac_avg": 0.004325927700847387, + "policy/entropy_avg": 0.1800684928894043, + "step": 132, + "val/clipfrac_avg": 0.07538644224405289, + "val/num_eos_tokens": 210, + "val/ratio": 0.9881160259246826, + "val/ratio_var": 6.755861249985173e-05 + }, + { + "episode": 2128, + "epoch": 0.10747474747474747, + "eps": 0, + "loss/policy_avg": -0.04767410457134247, + "loss/value_avg": 0.7692283391952515, + "lr": 9.912832366166443e-06, + "objective/entropy": -45.32240295410156, + "objective/kl": 10.798791885375977, + "objective/non_score_reward": -0.5399396419525146, + "objective/rlhf_reward": 7.835060119628906, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0035242601297795773, + "policy/clipfrac_avg": 0.03283273056149483, + "policy/entropy_avg": 0.22318172454833984, + "step": 133, + "val/clipfrac_avg": 0.21977931261062622, + "val/num_eos_tokens": 493, + "val/ratio": 0.9933129549026489, + "val/ratio_var": 0.0021998623851686716 + }, + { + "episode": 2144, + "epoch": 0.10828282828282829, + "eps": 0, + "loss/policy_avg": -0.5322861075401306, + "loss/value_avg": 2.0256404876708984, + "lr": 9.910331596834399e-06, + "objective/entropy": -128.28472900390625, + "objective/kl": 12.162965774536133, + "objective/non_score_reward": -0.6081483364105225, + "objective/rlhf_reward": 9.395757675170898, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0009850980713963509, + "policy/clipfrac_avg": 0.021821077913045883, + "policy/entropy_avg": 0.12530064582824707, + "step": 134, + "val/clipfrac_avg": 0.3014891743659973, + "val/num_eos_tokens": 6, + "val/ratio": 0.9908243417739868, + "val/ratio_var": 0.00012693816097453237 + }, + { + "episode": 2160, + "epoch": 0.10909090909090909, + "eps": 0, + "loss/policy_avg": -0.3030897378921509, + "loss/value_avg": 2.1831560134887695, + "lr": 9.907795784955327e-06, + "objective/entropy": -63.13277053833008, + "objective/kl": 12.188176155090332, + "objective/non_score_reward": -0.6094088554382324, + "objective/rlhf_reward": 9.10934066772461, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009403219446539879, + "policy/clipfrac_avg": 0.014169535599648952, + "policy/entropy_avg": 0.15417695045471191, + "step": 135, + "val/clipfrac_avg": 0.24678541719913483, + "val/num_eos_tokens": 481, + "val/ratio": 0.9922115802764893, + "val/ratio_var": 0.00014833426394034177 + }, + { + "episode": 2176, + "epoch": 0.1098989898989899, + "eps": 0, + "loss/policy_avg": -0.8595305681228638, + "loss/value_avg": 2.5992929935455322, + "lr": 9.905224948626033e-06, + "objective/entropy": -103.9739990234375, + "objective/kl": 10.518301010131836, + "objective/non_score_reward": -0.5259150862693787, + "objective/rlhf_reward": 10.681116104125977, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007561782840639353, + "policy/clipfrac_avg": 0.0030004261061549187, + "policy/entropy_avg": 0.18250799179077148, + "step": 136, + "val/clipfrac_avg": 0.35328760743141174, + "val/num_eos_tokens": 146, + "val/ratio": 0.9901064038276672, + "val/ratio_var": 6.593725993297994e-05 + }, + { + "episode": 2192, + "epoch": 0.11070707070707071, + "eps": 0, + "loss/policy_avg": 0.48323243856430054, + "loss/value_avg": 2.3285460472106934, + "lr": 9.90261910619327e-06, + "objective/entropy": -123.25853729248047, + "objective/kl": 14.112113952636719, + "objective/non_score_reward": -0.705605685710907, + "objective/rlhf_reward": 8.704550743103027, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001142181921750307, + "policy/clipfrac_avg": 0.025173479691147804, + "policy/entropy_avg": 0.13131451606750488, + "step": 137, + "val/clipfrac_avg": 0.17816151678562164, + "val/num_eos_tokens": 190, + "val/ratio": 0.9913126230239868, + "val/ratio_var": 5.5185322707984596e-05 + }, + { + "episode": 2208, + "epoch": 0.11151515151515151, + "eps": 0, + "loss/policy_avg": 0.041312795132398605, + "loss/value_avg": 1.2238047122955322, + "lr": 9.899978276253617e-06, + "objective/entropy": -83.2667465209961, + "objective/kl": 14.988029479980469, + "objective/non_score_reward": -0.7494014501571655, + "objective/rlhf_reward": 8.887317657470703, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0032717131543904543, + "policy/clipfrac_avg": 0.034809306263923645, + "policy/entropy_avg": 0.19169330596923828, + "step": 138, + "val/clipfrac_avg": 0.12297101318836212, + "val/num_eos_tokens": 458, + "val/ratio": 0.9875979423522949, + "val/ratio_var": 0.00023365950619336218 + }, + { + "episode": 2224, + "epoch": 0.11232323232323232, + "eps": 0, + "loss/policy_avg": 0.10726683586835861, + "loss/value_avg": 1.2004472017288208, + "lr": 9.897302477653334e-06, + "objective/entropy": -151.2768096923828, + "objective/kl": 10.463726043701172, + "objective/non_score_reward": -0.5231863260269165, + "objective/rlhf_reward": 9.847908020019531, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0015597918536514044, + "policy/clipfrac_avg": 0.03354030102491379, + "policy/entropy_avg": 0.13694477081298828, + "step": 139, + "val/clipfrac_avg": 0.25066325068473816, + "val/num_eos_tokens": 461, + "val/ratio": 0.9888541102409363, + "val/ratio_var": 0.0001727374765323475 + }, + { + "episode": 2240, + "epoch": 0.11313131313131314, + "eps": 0, + "loss/policy_avg": -0.1315029263496399, + "loss/value_avg": 1.1189866065979004, + "lr": 9.894591729488243e-06, + "objective/entropy": -56.48613739013672, + "objective/kl": 12.376513481140137, + "objective/non_score_reward": -0.6188256740570068, + "objective/rlhf_reward": 7.490549564361572, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009117410518229008, + "policy/clipfrac_avg": 0.013704899698495865, + "policy/entropy_avg": 0.14188098907470703, + "step": 140, + "val/clipfrac_avg": 0.12030020356178284, + "val/num_eos_tokens": 115, + "val/ratio": 0.9916878938674927, + "val/ratio_var": 7.079278293531388e-05 + }, + { + "episode": 2256, + "epoch": 0.11393939393939394, + "eps": 0, + "loss/policy_avg": -0.031752824783325195, + "loss/value_avg": 0.9798139333724976, + "lr": 9.891846051103578e-06, + "objective/entropy": -100.32903289794922, + "objective/kl": 13.879895210266113, + "objective/non_score_reward": -0.6939947605133057, + "objective/rlhf_reward": 7.934910774230957, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008554073283448815, + "policy/clipfrac_avg": 0.01539595890790224, + "policy/entropy_avg": 0.14811944961547852, + "step": 141, + "val/clipfrac_avg": 0.11368511617183685, + "val/num_eos_tokens": 926, + "val/ratio": 0.9901875257492065, + "val/ratio_var": 6.501925963675603e-05 + }, + { + "episode": 2272, + "epoch": 0.11474747474747475, + "eps": 0, + "loss/policy_avg": 0.16777145862579346, + "loss/value_avg": 1.338819980621338, + "lr": 9.889065462093851e-06, + "objective/entropy": -61.576045989990234, + "objective/kl": 14.766319274902344, + "objective/non_score_reward": -0.7383160591125488, + "objective/rlhf_reward": 7.910121440887451, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0012796547962352633, + "policy/clipfrac_avg": 0.01897471211850643, + "policy/entropy_avg": 0.15843772888183594, + "step": 142, + "val/clipfrac_avg": 0.10001248121261597, + "val/num_eos_tokens": 250, + "val/ratio": 0.9893121123313904, + "val/ratio_var": 4.402951162774116e-05 + }, + { + "episode": 2288, + "epoch": 0.11555555555555555, + "eps": 0, + "loss/policy_avg": 0.2867237329483032, + "loss/value_avg": 2.6249804496765137, + "lr": 9.88624998230272e-06, + "objective/entropy": -124.786376953125, + "objective/kl": 16.068700790405273, + "objective/non_score_reward": -0.8034350872039795, + "objective/rlhf_reward": 8.337189674377441, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009665879188105464, + "policy/clipfrac_avg": 0.02759074792265892, + "policy/entropy_avg": 0.26673293113708496, + "step": 143, + "val/clipfrac_avg": 0.2618255913257599, + "val/num_eos_tokens": 87, + "val/ratio": 0.9918559789657593, + "val/ratio_var": 4.804971831617877e-05 + }, + { + "episode": 2304, + "epoch": 0.11636363636363636, + "eps": 0, + "loss/policy_avg": 0.6126020550727844, + "loss/value_avg": 1.5799771547317505, + "lr": 9.883399631822836e-06, + "objective/entropy": -264.2563171386719, + "objective/kl": 14.85019588470459, + "objective/non_score_reward": -0.7425098419189453, + "objective/rlhf_reward": 6.929365158081055, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007313055102713406, + "policy/clipfrac_avg": 0.03529048711061478, + "policy/entropy_avg": 0.1719745397567749, + "step": 144, + "val/clipfrac_avg": 0.16758260130882263, + "val/num_eos_tokens": 104, + "val/ratio": 0.9931252002716064, + "val/ratio_var": 4.071718649356626e-05 + }, + { + "episode": 2320, + "epoch": 0.11717171717171718, + "eps": 0, + "loss/policy_avg": 0.11925015598535538, + "loss/value_avg": 1.104990005493164, + "lr": 9.880514430995703e-06, + "objective/entropy": -43.337398529052734, + "objective/kl": 17.772851943969727, + "objective/non_score_reward": -0.8886426687240601, + "objective/rlhf_reward": 7.296904563903809, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008921563276089728, + "policy/clipfrac_avg": 0.012217779643833637, + "policy/entropy_avg": 0.17539596557617188, + "step": 145, + "val/clipfrac_avg": 0.2284112274646759, + "val/num_eos_tokens": 268, + "val/ratio": 0.9908536672592163, + "val/ratio_var": 6.429313361877576e-05 + }, + { + "episode": 2336, + "epoch": 0.11797979797979798, + "eps": 0, + "loss/policy_avg": 0.08043037354946136, + "loss/value_avg": 1.5215954780578613, + "lr": 9.87759440041154e-06, + "objective/entropy": -95.0362777709961, + "objective/kl": 13.699361801147461, + "objective/non_score_reward": -0.684968113899231, + "objective/rlhf_reward": 10.244718551635742, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.015414729714393616, + "policy/clipfrac_avg": 0.041790302842855453, + "policy/entropy_avg": 0.11050856113433838, + "step": 146, + "val/clipfrac_avg": 0.18847554922103882, + "val/num_eos_tokens": 473, + "val/ratio": 0.990708589553833, + "val/ratio_var": 0.00013217887317296118 + }, + { + "episode": 2352, + "epoch": 0.11878787878787879, + "eps": 0, + "loss/policy_avg": -0.0913308784365654, + "loss/value_avg": 1.0237717628479004, + "lr": 9.874639560909118e-06, + "objective/entropy": -81.37150573730469, + "objective/kl": 15.634238243103027, + "objective/non_score_reward": -0.7817119359970093, + "objective/rlhf_reward": 10.054224967956543, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001198675250634551, + "policy/clipfrac_avg": 0.00917242094874382, + "policy/entropy_avg": 0.14813309907913208, + "step": 147, + "val/clipfrac_avg": 0.20073412358760834, + "val/num_eos_tokens": 175, + "val/ratio": 0.9932203888893127, + "val/ratio_var": 0.00023357552709057927 + }, + { + "episode": 2368, + "epoch": 0.1195959595959596, + "eps": 0, + "loss/policy_avg": -0.0016996562480926514, + "loss/value_avg": 0.8599748611450195, + "lr": 9.871649933575636e-06, + "objective/entropy": -58.469085693359375, + "objective/kl": 18.754764556884766, + "objective/non_score_reward": -0.937738299369812, + "objective/rlhf_reward": 7.876714706420898, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0012335460633039474, + "policy/clipfrac_avg": 0.015984678640961647, + "policy/entropy_avg": 0.2137432098388672, + "step": 148, + "val/clipfrac_avg": 0.2339939922094345, + "val/num_eos_tokens": 339, + "val/ratio": 0.9876729249954224, + "val/ratio_var": 0.0001175389188574627 + }, + { + "episode": 2384, + "epoch": 0.1204040404040404, + "eps": 0, + "loss/policy_avg": -0.3132669925689697, + "loss/value_avg": 0.7471304535865784, + "lr": 9.868625539746545e-06, + "objective/entropy": -153.42767333984375, + "objective/kl": 17.043121337890625, + "objective/non_score_reward": -0.8521560430526733, + "objective/rlhf_reward": 7.327531337738037, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0007452386198565364, + "policy/clipfrac_avg": 0.004154819063842297, + "policy/entropy_avg": 0.13955670595169067, + "step": 149, + "val/clipfrac_avg": 0.27733689546585083, + "val/num_eos_tokens": 543, + "val/ratio": 0.9911587238311768, + "val/ratio_var": 4.037221151520498e-05 + }, + { + "episode": 2400, + "epoch": 0.12121212121212122, + "eps": 0, + "loss/policy_avg": 0.5823547840118408, + "loss/value_avg": 2.283393144607544, + "lr": 9.86556640100541e-06, + "objective/entropy": -134.21249389648438, + "objective/kl": 15.889145851135254, + "objective/non_score_reward": -0.7944573163986206, + "objective/rlhf_reward": 8.15476131439209, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008621187880635262, + "policy/clipfrac_avg": 0.02879197522997856, + "policy/entropy_avg": 0.16280102729797363, + "step": 150, + "val/clipfrac_avg": 0.3507724404335022, + "val/num_eos_tokens": 742, + "val/ratio": 0.9914773106575012, + "val/ratio_var": 6.432795635191724e-05 + }, + { + "episode": 2416, + "epoch": 0.12202020202020201, + "eps": 0, + "loss/policy_avg": 0.00687573105096817, + "loss/value_avg": 0.5364193916320801, + "lr": 9.862472539183757e-06, + "objective/entropy": -73.51275634765625, + "objective/kl": 16.739295959472656, + "objective/non_score_reward": -0.8369647264480591, + "objective/rlhf_reward": 10.03412914276123, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008029645541682839, + "policy/clipfrac_avg": 0.007894085720181465, + "policy/entropy_avg": 0.15464520454406738, + "step": 151, + "val/clipfrac_avg": 0.25386878848075867, + "val/num_eos_tokens": 33, + "val/ratio": 0.9904509782791138, + "val/ratio_var": 2.9840115530532785e-05 + }, + { + "episode": 2432, + "epoch": 0.12282828282828283, + "eps": 0, + "loss/policy_avg": 0.46432220935821533, + "loss/value_avg": 2.502655267715454, + "lr": 9.859343976360904e-06, + "objective/entropy": -136.72979736328125, + "objective/kl": 14.286821365356445, + "objective/non_score_reward": -0.7143411040306091, + "objective/rlhf_reward": 9.172377586364746, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011772194411605597, + "policy/clipfrac_avg": 0.016035620123147964, + "policy/entropy_avg": 0.1345546841621399, + "step": 152, + "val/clipfrac_avg": 0.286276638507843, + "val/num_eos_tokens": 619, + "val/ratio": 0.9900628328323364, + "val/ratio_var": 7.126816490199417e-05 + }, + { + "episode": 2448, + "epoch": 0.12363636363636364, + "eps": 0, + "loss/policy_avg": 0.09378673136234283, + "loss/value_avg": 3.2477402687072754, + "lr": 9.85618073486382e-06, + "objective/entropy": -138.69039916992188, + "objective/kl": 16.09568214416504, + "objective/non_score_reward": -0.8047841191291809, + "objective/rlhf_reward": 8.843653678894043, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0012001043651252985, + "policy/clipfrac_avg": 0.0180644653737545, + "policy/entropy_avg": 0.149375319480896, + "step": 153, + "val/clipfrac_avg": 0.2985517382621765, + "val/num_eos_tokens": 433, + "val/ratio": 0.9893068671226501, + "val/ratio_var": 6.621519423788413e-05 + }, + { + "episode": 2464, + "epoch": 0.12444444444444444, + "eps": 0, + "loss/policy_avg": -0.3594944477081299, + "loss/value_avg": 2.61572527885437, + "lr": 9.852982837266955e-06, + "objective/entropy": -80.84510803222656, + "objective/kl": 19.26645278930664, + "objective/non_score_reward": -0.963322639465332, + "objective/rlhf_reward": 8.341363906860352, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007789209485054016, + "policy/clipfrac_avg": 0.006222645752131939, + "policy/entropy_avg": 0.12923431396484375, + "step": 154, + "val/clipfrac_avg": 0.14149022102355957, + "val/num_eos_tokens": 65, + "val/ratio": 0.9912763237953186, + "val/ratio_var": 5.006176434108056e-05 + }, + { + "episode": 2480, + "epoch": 0.12525252525252525, + "eps": 0, + "loss/policy_avg": 0.13503910601139069, + "loss/value_avg": 1.3053985834121704, + "lr": 9.849750306392085e-06, + "objective/entropy": -114.74142456054688, + "objective/kl": 16.327844619750977, + "objective/non_score_reward": -0.8163923025131226, + "objective/rlhf_reward": 9.566420555114746, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.001046534744091332, + "policy/clipfrac_avg": 0.01326068863272667, + "policy/entropy_avg": 0.13961029052734375, + "step": 155, + "val/clipfrac_avg": 0.12556566298007965, + "val/num_eos_tokens": 24, + "val/ratio": 0.9914346933364868, + "val/ratio_var": 8.751903078518808e-05 + }, + { + "episode": 2496, + "epoch": 0.12606060606060607, + "eps": 0, + "loss/policy_avg": -0.2862533926963806, + "loss/value_avg": 1.4972138404846191, + "lr": 9.846483165308142e-06, + "objective/entropy": -51.69160461425781, + "objective/kl": 14.01751708984375, + "objective/non_score_reward": -0.7008758783340454, + "objective/rlhf_reward": 9.084280014038086, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0020042983815073967, + "policy/clipfrac_avg": 0.025917891412973404, + "policy/entropy_avg": 0.18289995193481445, + "step": 156, + "val/clipfrac_avg": 0.07307614386081696, + "val/num_eos_tokens": 89, + "val/ratio": 0.988426923751831, + "val/ratio_var": 0.00010893259604927152 + }, + { + "episode": 2512, + "epoch": 0.12686868686868688, + "eps": 0, + "loss/policy_avg": 0.08509957790374756, + "loss/value_avg": 1.0140475034713745, + "lr": 9.843181437331055e-06, + "objective/entropy": -38.66559600830078, + "objective/kl": 18.988895416259766, + "objective/non_score_reward": -0.9494448304176331, + "objective/rlhf_reward": 8.070086479187012, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.004971734713762999, + "policy/clipfrac_avg": 0.017674520611763, + "policy/entropy_avg": 0.19965362548828125, + "step": 157, + "val/clipfrac_avg": 0.1631356030702591, + "val/num_eos_tokens": 85, + "val/ratio": 0.9906042218208313, + "val/ratio_var": 0.00015353818889707327 + }, + { + "episode": 2528, + "epoch": 0.12767676767676767, + "eps": 0, + "loss/policy_avg": -0.30014684796333313, + "loss/value_avg": 2.4983510971069336, + "lr": 9.839845146023583e-06, + "objective/entropy": -81.05842590332031, + "objective/kl": 14.4805908203125, + "objective/non_score_reward": -0.724029541015625, + "objective/rlhf_reward": 9.068939208984375, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0011242629261687398, + "policy/clipfrac_avg": 0.03127668797969818, + "policy/entropy_avg": 0.18725228309631348, + "step": 158, + "val/clipfrac_avg": 0.15103457868099213, + "val/num_eos_tokens": 3, + "val/ratio": 0.9933140277862549, + "val/ratio_var": 0.00010324850882170722 + }, + { + "episode": 2544, + "epoch": 0.12848484848484848, + "eps": 0, + "loss/policy_avg": 0.03238319233059883, + "loss/value_avg": 0.9245697855949402, + "lr": 9.836474315195148e-06, + "objective/entropy": -75.23670196533203, + "objective/kl": 18.89221954345703, + "objective/non_score_reward": -0.9446109533309937, + "objective/rlhf_reward": 9.160857200622559, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.005313557107001543, + "policy/clipfrac_avg": 0.017709676176309586, + "policy/entropy_avg": 0.2630615234375, + "step": 159, + "val/clipfrac_avg": 0.09926828742027283, + "val/num_eos_tokens": 214, + "val/ratio": 0.9845870137214661, + "val/ratio_var": 0.0004384346539154649 + }, + { + "episode": 2560, + "epoch": 0.1292929292929293, + "eps": 0, + "loss/policy_avg": -0.00330284982919693, + "loss/value_avg": 1.9382867813110352, + "lr": 9.833068968901657e-06, + "objective/entropy": -58.01062774658203, + "objective/kl": 14.313985824584961, + "objective/non_score_reward": -0.7156991958618164, + "objective/rlhf_reward": 7.667113304138184, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008890245808288455, + "policy/clipfrac_avg": 0.00987298134714365, + "policy/entropy_avg": 0.19803333282470703, + "step": 160, + "val/clipfrac_avg": 0.11537374556064606, + "val/num_eos_tokens": 85, + "val/ratio": 0.9886167049407959, + "val/ratio_var": 5.6686280004214495e-05 + }, + { + "episode": 2576, + "epoch": 0.1301010101010101, + "eps": 0, + "loss/policy_avg": 0.15009576082229614, + "loss/value_avg": 1.3036422729492188, + "lr": 9.829629131445342e-06, + "objective/entropy": -105.31230163574219, + "objective/kl": 14.487106323242188, + "objective/non_score_reward": -0.7243553400039673, + "objective/rlhf_reward": 9.494394302368164, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0008457785006612539, + "policy/clipfrac_avg": 0.01930004358291626, + "policy/entropy_avg": 0.16261720657348633, + "step": 161, + "val/clipfrac_avg": 0.20280539989471436, + "val/num_eos_tokens": 333, + "val/ratio": 0.9908741116523743, + "val/ratio_var": 8.265120413852856e-05 + }, + { + "episode": 2592, + "epoch": 0.13090909090909092, + "eps": 0, + "loss/policy_avg": 0.7536355257034302, + "loss/value_avg": 2.164299249649048, + "lr": 9.826154827374578e-06, + "objective/entropy": -119.47211456298828, + "objective/kl": 16.28435516357422, + "objective/non_score_reward": -0.8142178058624268, + "objective/rlhf_reward": 6.927969455718994, + "objective/scores": 7.75, + "policy/approxkl_avg": 0.0012012721272185445, + "policy/clipfrac_avg": 0.03371775150299072, + "policy/entropy_avg": 0.2003459930419922, + "step": 162, + "val/clipfrac_avg": 0.365829735994339, + "val/num_eos_tokens": 737, + "val/ratio": 0.9901646971702576, + "val/ratio_var": 9.40158570301719e-05 + }, + { + "episode": 2608, + "epoch": 0.1317171717171717, + "eps": 0, + "loss/policy_avg": -0.27083319425582886, + "loss/value_avg": 1.6828413009643555, + "lr": 9.822646081483713e-06, + "objective/entropy": -88.83195495605469, + "objective/kl": 17.124767303466797, + "objective/non_score_reward": -0.8562383651733398, + "objective/rlhf_reward": 9.327356338500977, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007394744316115975, + "policy/clipfrac_avg": 0.0198641587048769, + "policy/entropy_avg": 0.13005447387695312, + "step": 163, + "val/clipfrac_avg": 0.3026716709136963, + "val/num_eos_tokens": 512, + "val/ratio": 0.9908427000045776, + "val/ratio_var": 6.518982991110533e-05 + }, + { + "episode": 2624, + "epoch": 0.13252525252525252, + "eps": 0, + "loss/policy_avg": -0.37486350536346436, + "loss/value_avg": 2.394012689590454, + "lr": 9.819102918812887e-06, + "objective/entropy": -210.48764038085938, + "objective/kl": 15.069713592529297, + "objective/non_score_reward": -0.7534856796264648, + "objective/rlhf_reward": 7.668389320373535, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.000925661763176322, + "policy/clipfrac_avg": 0.02412327565252781, + "policy/entropy_avg": 0.16263806819915771, + "step": 164, + "val/clipfrac_avg": 0.14958947896957397, + "val/num_eos_tokens": 676, + "val/ratio": 0.9890938997268677, + "val/ratio_var": 0.0001162591579486616 + }, + { + "episode": 2640, + "epoch": 0.13333333333333333, + "eps": 0, + "loss/policy_avg": -0.3021676242351532, + "loss/value_avg": 0.7970246076583862, + "lr": 9.815525364647853e-06, + "objective/entropy": -53.14797592163086, + "objective/kl": 16.533164978027344, + "objective/non_score_reward": -0.8266582489013672, + "objective/rlhf_reward": 8.817873001098633, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0009110436076298356, + "policy/clipfrac_avg": 0.005712536629289389, + "policy/entropy_avg": 0.17499542236328125, + "step": 165, + "val/clipfrac_avg": 0.15537136793136597, + "val/num_eos_tokens": 536, + "val/ratio": 0.9888813495635986, + "val/ratio_var": 3.097565422649495e-05 + }, + { + "episode": 2656, + "epoch": 0.13414141414141414, + "eps": 0, + "loss/policy_avg": 0.10213343799114227, + "loss/value_avg": 0.985383927822113, + "lr": 9.811913444519803e-06, + "objective/entropy": -61.10470962524414, + "objective/kl": 12.42782974243164, + "objective/non_score_reward": -0.6213915348052979, + "objective/rlhf_reward": 8.554389953613281, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008850512094795704, + "policy/clipfrac_avg": 0.013360471464693546, + "policy/entropy_avg": 0.1629645824432373, + "step": 166, + "val/clipfrac_avg": 0.25333312153816223, + "val/num_eos_tokens": 73, + "val/ratio": 0.9904332160949707, + "val/ratio_var": 7.648977043572813e-05 + }, + { + "episode": 2672, + "epoch": 0.13494949494949496, + "eps": 0, + "loss/policy_avg": 0.5110056400299072, + "loss/value_avg": 1.0773067474365234, + "lr": 9.808267184205182e-06, + "objective/entropy": -38.80767059326172, + "objective/kl": 14.755228996276855, + "objective/non_score_reward": -0.7377614974975586, + "objective/rlhf_reward": 7.594269752502441, + "objective/scores": 8.3125, + "policy/approxkl_avg": 0.0033133821561932564, + "policy/clipfrac_avg": 0.012325381860136986, + "policy/entropy_avg": 0.20802688598632812, + "step": 167, + "val/clipfrac_avg": 0.166399747133255, + "val/num_eos_tokens": 280, + "val/ratio": 0.9878016710281372, + "val/ratio_var": 5.6116346968337893e-05 + }, + { + "episode": 2688, + "epoch": 0.13575757575757577, + "eps": 0, + "loss/policy_avg": -0.0654146745800972, + "loss/value_avg": 1.8866548538208008, + "lr": 9.804586609725499e-06, + "objective/entropy": -45.07624816894531, + "objective/kl": 14.000960350036621, + "objective/non_score_reward": -0.70004802942276, + "objective/rlhf_reward": 9.370265007019043, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0028054637368768454, + "policy/clipfrac_avg": 0.017689436674118042, + "policy/entropy_avg": 0.20137405395507812, + "step": 168, + "val/clipfrac_avg": 0.4012067914009094, + "val/num_eos_tokens": 219, + "val/ratio": 0.984307050704956, + "val/ratio_var": 0.0002625912893563509 + }, + { + "episode": 2704, + "epoch": 0.13656565656565656, + "eps": 0, + "loss/policy_avg": -0.26235154271125793, + "loss/value_avg": 0.9361153841018677, + "lr": 9.800871747347148e-06, + "objective/entropy": -29.569345474243164, + "objective/kl": 12.764520645141602, + "objective/non_score_reward": -0.6382260322570801, + "objective/rlhf_reward": 9.596149444580078, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0026277583092451096, + "policy/clipfrac_avg": 0.020652472972869873, + "policy/entropy_avg": 0.1861964464187622, + "step": 169, + "val/clipfrac_avg": 0.07669641077518463, + "val/num_eos_tokens": 114, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 0.0003732522891368717 + }, + { + "episode": 2720, + "epoch": 0.13737373737373737, + "eps": 0, + "loss/policy_avg": 0.0036516934633255005, + "loss/value_avg": 0.7808534502983093, + "lr": 9.797122623581217e-06, + "objective/entropy": -115.31767272949219, + "objective/kl": 14.564604759216309, + "objective/non_score_reward": -0.7282302379608154, + "objective/rlhf_reward": 8.541301727294922, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009797554230317473, + "policy/clipfrac_avg": 0.013546416535973549, + "policy/entropy_avg": 0.17569169402122498, + "step": 170, + "val/clipfrac_avg": 0.13324236869812012, + "val/num_eos_tokens": 24, + "val/ratio": 0.9880359172821045, + "val/ratio_var": 5.694741048500873e-05 + }, + { + "episode": 2736, + "epoch": 0.13818181818181818, + "eps": 0, + "loss/policy_avg": -0.3128410577774048, + "loss/value_avg": 0.6949845552444458, + "lr": 9.793339265183303e-06, + "objective/entropy": -65.40606689453125, + "objective/kl": 17.099538803100586, + "objective/non_score_reward": -0.854977011680603, + "objective/rlhf_reward": 8.246585845947266, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0009173300350084901, + "policy/clipfrac_avg": 0.011180122382938862, + "policy/entropy_avg": 0.17095184326171875, + "step": 171, + "val/clipfrac_avg": 0.09039227664470673, + "val/num_eos_tokens": 391, + "val/ratio": 0.9889956116676331, + "val/ratio_var": 5.041521944804117e-05 + }, + { + "episode": 2752, + "epoch": 0.138989898989899, + "eps": 0, + "loss/policy_avg": -0.06095901131629944, + "loss/value_avg": 0.6817218065261841, + "lr": 9.78952169915332e-06, + "objective/entropy": -149.44508361816406, + "objective/kl": 14.352700233459473, + "objective/non_score_reward": -0.7176350355148315, + "objective/rlhf_reward": 7.446427345275879, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.000751341343857348, + "policy/clipfrac_avg": 0.014955854043364525, + "policy/entropy_avg": 0.1538681983947754, + "step": 172, + "val/clipfrac_avg": 0.3020364046096802, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901411533355713, + "val/ratio_var": 5.242693077889271e-05 + }, + { + "episode": 2768, + "epoch": 0.1397979797979798, + "eps": 0, + "loss/policy_avg": -0.9073721170425415, + "loss/value_avg": 3.3096258640289307, + "lr": 9.785669952735294e-06, + "objective/entropy": -103.2147216796875, + "objective/kl": 18.063003540039062, + "objective/non_score_reward": -0.9031502604484558, + "objective/rlhf_reward": 10.32341194152832, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0011648546205833554, + "policy/clipfrac_avg": 0.0217579435557127, + "policy/entropy_avg": 0.18919309973716736, + "step": 173, + "val/clipfrac_avg": 0.36076119542121887, + "val/num_eos_tokens": 583, + "val/ratio": 0.9889928102493286, + "val/ratio_var": 0.00014562088472303003 + }, + { + "episode": 2784, + "epoch": 0.1406060606060606, + "eps": 0, + "loss/policy_avg": 0.9771312475204468, + "loss/value_avg": 3.1401190757751465, + "lr": 9.781784053417192e-06, + "objective/entropy": -144.74966430664062, + "objective/kl": 15.56573486328125, + "objective/non_score_reward": -0.7782866954803467, + "objective/rlhf_reward": 7.422885417938232, + "objective/scores": 8.1875, + "policy/approxkl_avg": 0.0008035623468458652, + "policy/clipfrac_avg": 0.03028782643377781, + "policy/entropy_avg": 0.13976645469665527, + "step": 174, + "val/clipfrac_avg": 0.326850026845932, + "val/num_eos_tokens": 455, + "val/ratio": 0.9907339215278625, + "val/ratio_var": 4.144087870372459e-05 + }, + { + "episode": 2800, + "epoch": 0.1414141414141414, + "eps": 0, + "loss/policy_avg": 0.28167733550071716, + "loss/value_avg": 1.6407313346862793, + "lr": 9.777864028930705e-06, + "objective/entropy": -106.90374755859375, + "objective/kl": 14.400297164916992, + "objective/non_score_reward": -0.7200148701667786, + "objective/rlhf_reward": 7.998735427856445, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0009868306806311011, + "policy/clipfrac_avg": 0.009237098507583141, + "policy/entropy_avg": 0.16825366020202637, + "step": 175, + "val/clipfrac_avg": 0.19442003965377808, + "val/num_eos_tokens": 59, + "val/ratio": 0.9899091720581055, + "val/ratio_var": 9.541668259771541e-05 + }, + { + "episode": 2816, + "epoch": 0.14222222222222222, + "eps": 0, + "loss/policy_avg": -0.15210214257240295, + "loss/value_avg": 1.2273833751678467, + "lr": 9.77390990725106e-06, + "objective/entropy": -60.392662048339844, + "objective/kl": 9.653593063354492, + "objective/non_score_reward": -0.48267969489097595, + "objective/rlhf_reward": 9.052475929260254, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010749915381893516, + "policy/clipfrac_avg": 0.00982856098562479, + "policy/entropy_avg": 0.16010570526123047, + "step": 176, + "val/clipfrac_avg": 0.11848568916320801, + "val/num_eos_tokens": 58, + "val/ratio": 0.9885416030883789, + "val/ratio_var": 5.872864130651578e-05 + }, + { + "episode": 2832, + "epoch": 0.14303030303030304, + "eps": 0, + "loss/policy_avg": 0.17394450306892395, + "loss/value_avg": 3.3872365951538086, + "lr": 9.76992171659682e-06, + "objective/entropy": -69.82381439208984, + "objective/kl": 11.461904525756836, + "objective/non_score_reward": -0.5730952620506287, + "objective/rlhf_reward": 11.044092178344727, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0013258052058517933, + "policy/clipfrac_avg": 0.02767137438058853, + "policy/entropy_avg": 0.15888667106628418, + "step": 177, + "val/clipfrac_avg": 0.25187912583351135, + "val/num_eos_tokens": 28, + "val/ratio": 0.9916998744010925, + "val/ratio_var": 7.523155363742262e-05 + }, + { + "episode": 2848, + "epoch": 0.14383838383838385, + "eps": 0, + "loss/policy_avg": -0.12190955877304077, + "loss/value_avg": 1.3829247951507568, + "lr": 9.76589948542968e-06, + "objective/entropy": -124.40007019042969, + "objective/kl": 16.81972312927246, + "objective/non_score_reward": -0.8409861326217651, + "objective/rlhf_reward": 10.639482498168945, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0010132414754480124, + "policy/clipfrac_avg": 0.008814403787255287, + "policy/entropy_avg": 0.15328598022460938, + "step": 178, + "val/clipfrac_avg": 0.2293124943971634, + "val/num_eos_tokens": 372, + "val/ratio": 0.9888865947723389, + "val/ratio_var": 7.727585034444928e-05 + }, + { + "episode": 2864, + "epoch": 0.14464646464646463, + "eps": 0, + "loss/policy_avg": -0.36637282371520996, + "loss/value_avg": 1.258082628250122, + "lr": 9.761843242454261e-06, + "objective/entropy": -97.340576171875, + "objective/kl": 12.127764701843262, + "objective/non_score_reward": -0.6063882112503052, + "objective/rlhf_reward": 8.141658782958984, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008368783164769411, + "policy/clipfrac_avg": 0.012604119256138802, + "policy/entropy_avg": 0.15296649932861328, + "step": 179, + "val/clipfrac_avg": 0.0866643637418747, + "val/num_eos_tokens": 53, + "val/ratio": 0.9895975589752197, + "val/ratio_var": 6.253716128412634e-05 + }, + { + "episode": 2880, + "epoch": 0.14545454545454545, + "eps": 0, + "loss/policy_avg": -0.7018676996231079, + "loss/value_avg": 2.839538097381592, + "lr": 9.757753016617917e-06, + "objective/entropy": -92.20721435546875, + "objective/kl": 12.383216857910156, + "objective/non_score_reward": -0.6191608309745789, + "objective/rlhf_reward": 10.091776847839355, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0010203049750998616, + "policy/clipfrac_avg": 0.020494675263762474, + "policy/entropy_avg": 0.14533498883247375, + "step": 180, + "val/clipfrac_avg": 0.19666892290115356, + "val/num_eos_tokens": 4, + "val/ratio": 0.9898504018783569, + "val/ratio_var": 8.984190208138898e-05 + }, + { + "episode": 2896, + "epoch": 0.14626262626262626, + "eps": 0, + "loss/policy_avg": -0.12757132947444916, + "loss/value_avg": 2.167374849319458, + "lr": 9.753628837110514e-06, + "objective/entropy": -76.44052124023438, + "objective/kl": 12.262140274047852, + "objective/non_score_reward": -0.6131070256233215, + "objective/rlhf_reward": 9.078298568725586, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0010697869583964348, + "policy/clipfrac_avg": 0.022361809387803078, + "policy/entropy_avg": 0.15213966369628906, + "step": 181, + "val/clipfrac_avg": 0.31748759746551514, + "val/num_eos_tokens": 158, + "val/ratio": 0.9903136491775513, + "val/ratio_var": 8.450200402876362e-05 + }, + { + "episode": 2912, + "epoch": 0.14707070707070707, + "eps": 0, + "loss/policy_avg": -0.030544668436050415, + "loss/value_avg": 1.8187932968139648, + "lr": 9.749470733364231e-06, + "objective/entropy": -149.43963623046875, + "objective/kl": 12.123230934143066, + "objective/non_score_reward": -0.6061615943908691, + "objective/rlhf_reward": 10.073525428771973, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008650605450384319, + "policy/clipfrac_avg": 0.029723089188337326, + "policy/entropy_avg": 0.14513269066810608, + "step": 182, + "val/clipfrac_avg": 0.1266334354877472, + "val/num_eos_tokens": 192, + "val/ratio": 0.9940528869628906, + "val/ratio_var": 9.982670599129051e-05 + }, + { + "episode": 2928, + "epoch": 0.1478787878787879, + "eps": 0, + "loss/policy_avg": 0.08868739008903503, + "loss/value_avg": 1.8785271644592285, + "lr": 9.745278735053345e-06, + "objective/entropy": -50.62373352050781, + "objective/kl": 15.383115768432617, + "objective/non_score_reward": -0.769155740737915, + "objective/rlhf_reward": 9.641000747680664, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.004477909300476313, + "policy/clipfrac_avg": 0.050629667937755585, + "policy/entropy_avg": 0.2141427993774414, + "step": 183, + "val/clipfrac_avg": 0.2094041109085083, + "val/num_eos_tokens": 2, + "val/ratio": 0.9843396544456482, + "val/ratio_var": 0.00037895137211307883 + }, + { + "episode": 2944, + "epoch": 0.1486868686868687, + "eps": 0, + "loss/policy_avg": -0.3627840578556061, + "loss/value_avg": 0.6112059354782104, + "lr": 9.74105287209402e-06, + "objective/entropy": -82.41788482666016, + "objective/kl": 15.159713745117188, + "objective/non_score_reward": -0.7579857110977173, + "objective/rlhf_reward": 7.968576431274414, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0010779302101582289, + "policy/clipfrac_avg": 0.011320716701447964, + "policy/entropy_avg": 0.17828106880187988, + "step": 184, + "val/clipfrac_avg": 0.15411992371082306, + "val/num_eos_tokens": 67, + "val/ratio": 0.9890798330307007, + "val/ratio_var": 9.126321674557403e-05 + }, + { + "episode": 2960, + "epoch": 0.1494949494949495, + "eps": 0, + "loss/policy_avg": -0.18067747354507446, + "loss/value_avg": 0.5258877277374268, + "lr": 9.736793174644106e-06, + "objective/entropy": -101.67889404296875, + "objective/kl": 12.352827072143555, + "objective/non_score_reward": -0.6176413297653198, + "objective/rlhf_reward": 9.17923355102539, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.011117171496152878, + "policy/clipfrac_avg": 0.017320796847343445, + "policy/entropy_avg": 0.15565109252929688, + "step": 185, + "val/clipfrac_avg": 0.19647085666656494, + "val/num_eos_tokens": 210, + "val/ratio": 0.984650731086731, + "val/ratio_var": 0.0008224447374232113 + }, + { + "episode": 2976, + "epoch": 0.1503030303030303, + "eps": 0, + "loss/policy_avg": 0.1885536015033722, + "loss/value_avg": 1.6109416484832764, + "lr": 9.732499673102895e-06, + "objective/entropy": -65.328369140625, + "objective/kl": 11.292556762695312, + "objective/non_score_reward": -0.5646278262138367, + "objective/rlhf_reward": 9.228340148925781, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0006467722705565393, + "policy/clipfrac_avg": 0.021413927897810936, + "policy/entropy_avg": 0.11110901832580566, + "step": 186, + "val/clipfrac_avg": 0.18745771050453186, + "val/num_eos_tokens": 246, + "val/ratio": 0.993016242980957, + "val/ratio_var": 3.980214387411252e-05 + }, + { + "episode": 2992, + "epoch": 0.1511111111111111, + "eps": 0, + "loss/policy_avg": 0.32616978883743286, + "loss/value_avg": 0.6681559681892395, + "lr": 9.728172398110935e-06, + "objective/entropy": -124.52982330322266, + "objective/kl": 16.007747650146484, + "objective/non_score_reward": -0.8003873825073242, + "objective/rlhf_reward": 7.242581844329834, + "objective/scores": 8.0625, + "policy/approxkl_avg": 0.0017935727955773473, + "policy/clipfrac_avg": 0.021759318187832832, + "policy/entropy_avg": 0.15468502044677734, + "step": 187, + "val/clipfrac_avg": 0.12101784348487854, + "val/num_eos_tokens": 21, + "val/ratio": 0.989864706993103, + "val/ratio_var": 5.842960308655165e-05 + }, + { + "episode": 3008, + "epoch": 0.15191919191919193, + "eps": 0, + "loss/policy_avg": 0.1749454289674759, + "loss/value_avg": 1.0790375471115112, + "lr": 9.723811380549792e-06, + "objective/entropy": -38.704368591308594, + "objective/kl": 14.366378784179688, + "objective/non_score_reward": -0.7183189988136292, + "objective/rlhf_reward": 7.746524810791016, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0014909141464158893, + "policy/clipfrac_avg": 0.025371305644512177, + "policy/entropy_avg": 0.15520191192626953, + "step": 188, + "val/clipfrac_avg": 0.1147574707865715, + "val/num_eos_tokens": 135, + "val/ratio": 0.989890456199646, + "val/ratio_var": 7.073572487570345e-05 + }, + { + "episode": 3024, + "epoch": 0.15272727272727274, + "eps": 0, + "loss/policy_avg": -0.5833231210708618, + "loss/value_avg": 3.6844418048858643, + "lr": 9.719416651541839e-06, + "objective/entropy": -104.98526000976562, + "objective/kl": 13.329991340637207, + "objective/non_score_reward": -0.6664996147155762, + "objective/rlhf_reward": 7.052250385284424, + "objective/scores": 7.71875, + "policy/approxkl_avg": 0.0009702666429802775, + "policy/clipfrac_avg": 0.03182617574930191, + "policy/entropy_avg": 0.1709080934524536, + "step": 189, + "val/clipfrac_avg": 0.15363654494285583, + "val/num_eos_tokens": 1103, + "val/ratio": 0.9898666143417358, + "val/ratio_var": 0.00010601755639072508 + }, + { + "episode": 3040, + "epoch": 0.15353535353535352, + "eps": 0, + "loss/policy_avg": -0.14998452365398407, + "loss/value_avg": 2.081155300140381, + "lr": 9.714988242450028e-06, + "objective/entropy": -87.40807342529297, + "objective/kl": 11.613686561584473, + "objective/non_score_reward": -0.5806843042373657, + "objective/rlhf_reward": 10.415409088134766, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0009950872045010328, + "policy/clipfrac_avg": 0.010731380432844162, + "policy/entropy_avg": 0.1969156265258789, + "step": 190, + "val/clipfrac_avg": 0.20180140435695648, + "val/num_eos_tokens": 140, + "val/ratio": 0.9887533783912659, + "val/ratio_var": 6.801114068366587e-05 + }, + { + "episode": 3056, + "epoch": 0.15434343434343434, + "eps": 0, + "loss/policy_avg": -0.24951934814453125, + "loss/value_avg": 0.500019907951355, + "lr": 9.710526184877667e-06, + "objective/entropy": -76.69435119628906, + "objective/kl": 11.947388648986816, + "objective/non_score_reward": -0.5973694324493408, + "objective/rlhf_reward": 9.894817352294922, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0009129929821938276, + "policy/clipfrac_avg": 0.006569317542016506, + "policy/entropy_avg": 0.16202569007873535, + "step": 191, + "val/clipfrac_avg": 0.13341541588306427, + "val/num_eos_tokens": 435, + "val/ratio": 0.9887447357177734, + "val/ratio_var": 5.932084968662821e-05 + }, + { + "episode": 3072, + "epoch": 0.15515151515151515, + "eps": 0, + "loss/policy_avg": 0.07577252388000488, + "loss/value_avg": 0.5789152979850769, + "lr": 9.706030510668202e-06, + "objective/entropy": -67.43034362792969, + "objective/kl": 13.22661304473877, + "objective/non_score_reward": -0.6613306403160095, + "objective/rlhf_reward": 10.838668823242188, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0012521710013970733, + "policy/clipfrac_avg": 0.011959927156567574, + "policy/entropy_avg": 0.16521143913269043, + "step": 192, + "val/clipfrac_avg": 0.1288057565689087, + "val/num_eos_tokens": 212, + "val/ratio": 0.9881963133811951, + "val/ratio_var": 7.64970391173847e-05 + }, + { + "episode": 3088, + "epoch": 0.15595959595959596, + "eps": 0, + "loss/policy_avg": -0.4162842333316803, + "loss/value_avg": 2.1575677394866943, + "lr": 9.70150125190498e-06, + "objective/entropy": -95.4016342163086, + "objective/kl": 10.532510757446289, + "objective/non_score_reward": -0.5266256332397461, + "objective/rlhf_reward": 11.563218116760254, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009733056649565697, + "policy/clipfrac_avg": 0.015730470418930054, + "policy/entropy_avg": 0.16752052307128906, + "step": 193, + "val/clipfrac_avg": 0.2311292290687561, + "val/num_eos_tokens": 344, + "val/ratio": 0.9886439442634583, + "val/ratio_var": 0.00013025107909925282 + }, + { + "episode": 3104, + "epoch": 0.15676767676767678, + "eps": 0, + "loss/policy_avg": 0.6562298536300659, + "loss/value_avg": 1.8542827367782593, + "lr": 9.696938440911025e-06, + "objective/entropy": -155.12109375, + "objective/kl": 11.387260437011719, + "objective/non_score_reward": -0.569362998008728, + "objective/rlhf_reward": 8.19040298461914, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0007924421224743128, + "policy/clipfrac_avg": 0.014981664717197418, + "policy/entropy_avg": 0.15529108047485352, + "step": 194, + "val/clipfrac_avg": 0.21850237250328064, + "val/num_eos_tokens": 38, + "val/ratio": 0.9898360967636108, + "val/ratio_var": 0.000115352297143545 + }, + { + "episode": 3120, + "epoch": 0.15757575757575756, + "eps": 0, + "loss/policy_avg": -0.23366859555244446, + "loss/value_avg": 1.7562166452407837, + "lr": 9.692342110248802e-06, + "objective/entropy": -108.78264617919922, + "objective/kl": 14.468893051147461, + "objective/non_score_reward": -0.7234447002410889, + "objective/rlhf_reward": 10.987492561340332, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0011592827504500747, + "policy/clipfrac_avg": 0.012323892675340176, + "policy/entropy_avg": 0.13213586807250977, + "step": 195, + "val/clipfrac_avg": 0.3065049946308136, + "val/num_eos_tokens": 191, + "val/ratio": 0.9917997121810913, + "val/ratio_var": 5.691219848813489e-05 + }, + { + "episode": 3136, + "epoch": 0.15838383838383838, + "eps": 0, + "loss/policy_avg": -0.12489986419677734, + "loss/value_avg": 0.3401644229888916, + "lr": 9.687712292719997e-06, + "objective/entropy": -72.25318908691406, + "objective/kl": 13.238128662109375, + "objective/non_score_reward": -0.6619064807891846, + "objective/rlhf_reward": 8.775592803955078, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008679552120156586, + "policy/clipfrac_avg": 0.010309899225831032, + "policy/entropy_avg": 0.16907691955566406, + "step": 196, + "val/clipfrac_avg": 0.11943725496530533, + "val/num_eos_tokens": 279, + "val/ratio": 0.9904986619949341, + "val/ratio_var": 6.888979987706989e-05 + }, + { + "episode": 3152, + "epoch": 0.1591919191919192, + "eps": 0, + "loss/policy_avg": -0.6271263360977173, + "loss/value_avg": 1.2400249242782593, + "lr": 9.683049021365267e-06, + "objective/entropy": -102.1280746459961, + "objective/kl": 15.960373878479004, + "objective/non_score_reward": -0.7980186939239502, + "objective/rlhf_reward": 9.143386840820312, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0009381669806316495, + "policy/clipfrac_avg": 0.015553677454590797, + "policy/entropy_avg": 0.1754164695739746, + "step": 197, + "val/clipfrac_avg": 0.1764695793390274, + "val/num_eos_tokens": 110, + "val/ratio": 0.9885835647583008, + "val/ratio_var": 5.869405867997557e-05 + }, + { + "episode": 3168, + "epoch": 0.16, + "eps": 0, + "loss/policy_avg": -0.038626529276371, + "loss/value_avg": 0.6332731246948242, + "lr": 9.678352329464018e-06, + "objective/entropy": -89.7190170288086, + "objective/kl": 11.87062931060791, + "objective/non_score_reward": -0.5935314893722534, + "objective/rlhf_reward": 9.843968391418457, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008885859278962016, + "policy/clipfrac_avg": 0.028107576072216034, + "policy/entropy_avg": 0.11816668510437012, + "step": 198, + "val/clipfrac_avg": 0.29144570231437683, + "val/num_eos_tokens": 283, + "val/ratio": 0.991458535194397, + "val/ratio_var": 6.830303755123168e-05 + }, + { + "episode": 3184, + "epoch": 0.16080808080808082, + "eps": 0, + "loss/policy_avg": 0.5430840849876404, + "loss/value_avg": 1.1472291946411133, + "lr": 9.673622250534155e-06, + "objective/entropy": -124.32833862304688, + "objective/kl": 11.320138931274414, + "objective/non_score_reward": -0.5660068988800049, + "objective/rlhf_reward": 8.316804885864258, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.000670759822241962, + "policy/clipfrac_avg": 0.009003499522805214, + "policy/entropy_avg": 0.1954498291015625, + "step": 199, + "val/clipfrac_avg": 0.1790837049484253, + "val/num_eos_tokens": 554, + "val/ratio": 0.9917429685592651, + "val/ratio_var": 4.731408989755437e-05 + }, + { + "episode": 3200, + "epoch": 0.16161616161616163, + "eps": 0, + "loss/policy_avg": -0.028545048087835312, + "loss/value_avg": 0.8721814155578613, + "lr": 9.66885881833186e-06, + "objective/entropy": -106.52888488769531, + "objective/kl": 12.4759521484375, + "objective/non_score_reward": -0.623797595500946, + "objective/rlhf_reward": 7.981671333312988, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009805381996557117, + "policy/clipfrac_avg": 0.011222105473279953, + "policy/entropy_avg": 0.1516423225402832, + "step": 200, + "val/clipfrac_avg": 0.11438994109630585, + "val/num_eos_tokens": 13, + "val/ratio": 0.9880229830741882, + "val/ratio_var": 0.0001519431680208072 + }, + { + "episode": 3216, + "epoch": 0.16242424242424242, + "eps": 0, + "loss/policy_avg": 0.03965132683515549, + "loss/value_avg": 0.8802922964096069, + "lr": 9.664062066851325e-06, + "objective/entropy": -55.07182693481445, + "objective/kl": 10.964399337768555, + "objective/non_score_reward": -0.5482199192047119, + "objective/rlhf_reward": 8.709592819213867, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0027661495842039585, + "policy/clipfrac_avg": 0.022680213674902916, + "policy/entropy_avg": 0.15708684921264648, + "step": 201, + "val/clipfrac_avg": 0.2466743290424347, + "val/num_eos_tokens": 274, + "val/ratio": 0.9882746934890747, + "val/ratio_var": 0.00016298489936161786 + }, + { + "episode": 3232, + "epoch": 0.16323232323232323, + "eps": 0, + "loss/policy_avg": 0.31882551312446594, + "loss/value_avg": 1.1626311540603638, + "lr": 9.659232030324538e-06, + "objective/entropy": -98.02118682861328, + "objective/kl": 12.811698913574219, + "objective/non_score_reward": -0.6405849456787109, + "objective/rlhf_reward": 7.718790054321289, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0010725643951445818, + "policy/clipfrac_avg": 0.010770853608846664, + "policy/entropy_avg": 0.1952628493309021, + "step": 202, + "val/clipfrac_avg": 0.14545708894729614, + "val/num_eos_tokens": 143, + "val/ratio": 0.987186074256897, + "val/ratio_var": 0.00014847182319499552 + }, + { + "episode": 3248, + "epoch": 0.16404040404040404, + "eps": 0, + "loss/policy_avg": -0.036153364926576614, + "loss/value_avg": 0.21554255485534668, + "lr": 9.654368743221022e-06, + "objective/entropy": -72.79200744628906, + "objective/kl": 16.778663635253906, + "objective/non_score_reward": -0.8389331698417664, + "objective/rlhf_reward": 7.258723258972168, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009537228033877909, + "policy/clipfrac_avg": 0.0074140760116279125, + "policy/entropy_avg": 0.1759014129638672, + "step": 203, + "val/clipfrac_avg": 0.08175989985466003, + "val/num_eos_tokens": 82, + "val/ratio": 0.9885472059249878, + "val/ratio_var": 2.7167381631443277e-05 + }, + { + "episode": 3264, + "epoch": 0.16484848484848486, + "eps": 0, + "loss/policy_avg": -0.25860804319381714, + "loss/value_avg": 1.3965643644332886, + "lr": 9.649472240247588e-06, + "objective/entropy": -123.76312255859375, + "objective/kl": 14.448421478271484, + "objective/non_score_reward": -0.7224210500717163, + "objective/rlhf_reward": 9.508047103881836, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007930218707770109, + "policy/clipfrac_avg": 0.010996568016707897, + "policy/entropy_avg": 0.1407322883605957, + "step": 204, + "val/clipfrac_avg": 0.1387026160955429, + "val/num_eos_tokens": 154, + "val/ratio": 0.991324245929718, + "val/ratio_var": 6.240070069907233e-05 + }, + { + "episode": 3280, + "epoch": 0.16565656565656567, + "eps": 0, + "loss/policy_avg": -0.58719801902771, + "loss/value_avg": 1.1743042469024658, + "lr": 9.6445425563481e-06, + "objective/entropy": -65.26087951660156, + "objective/kl": 11.825948715209961, + "objective/non_score_reward": -0.591297447681427, + "objective/rlhf_reward": 9.178234100341797, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0007801909814588726, + "policy/clipfrac_avg": 0.005785930901765823, + "policy/entropy_avg": 0.14145886898040771, + "step": 205, + "val/clipfrac_avg": 0.25026363134384155, + "val/num_eos_tokens": 16, + "val/ratio": 0.9906296133995056, + "val/ratio_var": 6.658971688011661e-05 + }, + { + "episode": 3296, + "epoch": 0.16646464646464645, + "eps": 0, + "loss/policy_avg": 0.26090189814567566, + "loss/value_avg": 0.48005229234695435, + "lr": 9.639579726703205e-06, + "objective/entropy": -76.86801147460938, + "objective/kl": 14.593581199645996, + "objective/non_score_reward": -0.7296790480613708, + "objective/rlhf_reward": 6.567195892333984, + "objective/scores": 7.3125, + "policy/approxkl_avg": 0.0008751621353439987, + "policy/clipfrac_avg": 0.00959798227995634, + "policy/entropy_avg": 0.16647958755493164, + "step": 206, + "val/clipfrac_avg": 0.2793257236480713, + "val/num_eos_tokens": 509, + "val/ratio": 0.9888375997543335, + "val/ratio_var": 7.126013224478811e-05 + }, + { + "episode": 3312, + "epoch": 0.16727272727272727, + "eps": 0, + "loss/policy_avg": 0.10253681987524033, + "loss/value_avg": 1.0820584297180176, + "lr": 9.63458378673011e-06, + "objective/entropy": -58.051673889160156, + "objective/kl": 9.699974060058594, + "objective/non_score_reward": -0.4849987328052521, + "objective/rlhf_reward": 8.22984504699707, + "objective/scores": 8.6875, + "policy/approxkl_avg": 0.000850443379022181, + "policy/clipfrac_avg": 0.008085163310170174, + "policy/entropy_avg": 0.16680526733398438, + "step": 207, + "val/clipfrac_avg": 0.2619618773460388, + "val/num_eos_tokens": 61, + "val/ratio": 0.9909214973449707, + "val/ratio_var": 6.327883602352813e-05 + }, + { + "episode": 3328, + "epoch": 0.16808080808080808, + "eps": 0, + "loss/policy_avg": 0.1208038404583931, + "loss/value_avg": 0.7759054899215698, + "lr": 9.6295547720823e-06, + "objective/entropy": -32.6884651184082, + "objective/kl": 14.656696319580078, + "objective/non_score_reward": -0.7328348159790039, + "objective/rlhf_reward": 9.114821434020996, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0014994569355621934, + "policy/clipfrac_avg": 0.012033997103571892, + "policy/entropy_avg": 0.22088956832885742, + "step": 208, + "val/clipfrac_avg": 0.18890069425106049, + "val/num_eos_tokens": 364, + "val/ratio": 0.9868152737617493, + "val/ratio_var": 0.000158175069373101 + }, + { + "episode": 3344, + "epoch": 0.1688888888888889, + "eps": 0, + "loss/policy_avg": 0.011440183967351913, + "loss/value_avg": 0.2232610136270523, + "lr": 9.624492718649306e-06, + "objective/entropy": -99.2800521850586, + "objective/kl": 15.448038101196289, + "objective/non_score_reward": -0.7724019289016724, + "objective/rlhf_reward": 7.688535690307617, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0008467785082757473, + "policy/clipfrac_avg": 0.00758355762809515, + "policy/entropy_avg": 0.15184712409973145, + "step": 209, + "val/clipfrac_avg": 0.05371936410665512, + "val/num_eos_tokens": 255, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.036638347315602e-05 + }, + { + "episode": 3360, + "epoch": 0.1696969696969697, + "eps": 0, + "loss/policy_avg": 0.05170050263404846, + "loss/value_avg": 0.7091935276985168, + "lr": 9.619397662556434e-06, + "objective/entropy": -74.2175521850586, + "objective/kl": 13.331558227539062, + "objective/non_score_reward": -0.666577935218811, + "objective/rlhf_reward": 7.8334221839904785, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0010089495917782187, + "policy/clipfrac_avg": 0.009919734671711922, + "policy/entropy_avg": 0.19567251205444336, + "step": 210, + "val/clipfrac_avg": 0.2976298928260803, + "val/num_eos_tokens": 599, + "val/ratio": 0.9870831966400146, + "val/ratio_var": 6.72663445584476e-05 + }, + { + "episode": 3376, + "epoch": 0.1705050505050505, + "eps": 0, + "loss/policy_avg": -0.4411523938179016, + "loss/value_avg": 0.5515732765197754, + "lr": 9.61426964016452e-06, + "objective/entropy": -130.67494201660156, + "objective/kl": 13.624598503112793, + "objective/non_score_reward": -0.6812299489974976, + "objective/rlhf_reward": 9.256270408630371, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007514972821809351, + "policy/clipfrac_avg": 0.0054230415262281895, + "policy/entropy_avg": 0.15925180912017822, + "step": 211, + "val/clipfrac_avg": 0.2941030263900757, + "val/num_eos_tokens": 550, + "val/ratio": 0.9901284575462341, + "val/ratio_var": 5.974218584015034e-05 + }, + { + "episode": 3392, + "epoch": 0.1713131313131313, + "eps": 0, + "loss/policy_avg": 0.004086721688508987, + "loss/value_avg": 0.2312389463186264, + "lr": 9.609108688069655e-06, + "objective/entropy": -25.59563446044922, + "objective/kl": 13.889019012451172, + "objective/non_score_reward": -0.6944509744644165, + "objective/rlhf_reward": 7.411017417907715, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0009575081057846546, + "policy/clipfrac_avg": 0.009083281271159649, + "policy/entropy_avg": 0.15931987762451172, + "step": 212, + "val/clipfrac_avg": 0.17202627658843994, + "val/num_eos_tokens": 129, + "val/ratio": 0.9893661737442017, + "val/ratio_var": 3.219742211513221e-05 + }, + { + "episode": 3408, + "epoch": 0.17212121212121212, + "eps": 0, + "loss/policy_avg": 0.027073439210653305, + "loss/value_avg": 0.22074271738529205, + "lr": 9.603914843102941e-06, + "objective/entropy": -68.1807861328125, + "objective/kl": 16.511375427246094, + "objective/non_score_reward": -0.8255687355995178, + "objective/rlhf_reward": 7.791618347167969, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0009854808449745178, + "policy/clipfrac_avg": 0.005424494855105877, + "policy/entropy_avg": 0.17797672748565674, + "step": 213, + "val/clipfrac_avg": 0.06424004584550858, + "val/num_eos_tokens": 11, + "val/ratio": 0.9874800443649292, + "val/ratio_var": 5.017150397179648e-05 + }, + { + "episode": 3424, + "epoch": 0.17292929292929293, + "eps": 0, + "loss/policy_avg": -0.22705355286598206, + "loss/value_avg": 0.47900986671447754, + "lr": 9.598688142330216e-06, + "objective/entropy": -147.718994140625, + "objective/kl": 14.358128547668457, + "objective/non_score_reward": -0.7179064750671387, + "objective/rlhf_reward": 9.012561798095703, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008998280391097069, + "policy/clipfrac_avg": 0.006018335931003094, + "policy/entropy_avg": 0.16658711433410645, + "step": 214, + "val/clipfrac_avg": 0.08306095004081726, + "val/num_eos_tokens": 616, + "val/ratio": 0.9889283180236816, + "val/ratio_var": 7.233618816826493e-05 + }, + { + "episode": 3440, + "epoch": 0.17373737373737375, + "eps": 0, + "loss/policy_avg": -0.14958789944648743, + "loss/value_avg": 2.3287415504455566, + "lr": 9.593428623051793e-06, + "objective/entropy": -91.03656768798828, + "objective/kl": 9.831127166748047, + "objective/non_score_reward": -0.4915563762187958, + "objective/rlhf_reward": 9.992818832397461, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0010748449712991714, + "policy/clipfrac_avg": 0.022500766441226006, + "policy/entropy_avg": 0.19421768188476562, + "step": 215, + "val/clipfrac_avg": 0.27161675691604614, + "val/num_eos_tokens": 249, + "val/ratio": 0.9880467653274536, + "val/ratio_var": 0.00019949306442867965 + }, + { + "episode": 3456, + "epoch": 0.17454545454545456, + "eps": 0, + "loss/policy_avg": 0.2554932236671448, + "loss/value_avg": 1.375806212425232, + "lr": 9.588136322802194e-06, + "objective/entropy": -65.73722839355469, + "objective/kl": 11.43238639831543, + "objective/non_score_reward": -0.5716192722320557, + "objective/rlhf_reward": 9.635412216186523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0008728803368285298, + "policy/clipfrac_avg": 0.016023337841033936, + "policy/entropy_avg": 0.16358375549316406, + "step": 216, + "val/clipfrac_avg": 0.12904679775238037, + "val/num_eos_tokens": 12, + "val/ratio": 0.9907691478729248, + "val/ratio_var": 4.799722955795005e-05 + }, + { + "episode": 3472, + "epoch": 0.17535353535353534, + "eps": 0, + "loss/policy_avg": 0.39980143308639526, + "loss/value_avg": 0.743119478225708, + "lr": 9.582811279349881e-06, + "objective/entropy": -72.137451171875, + "objective/kl": 13.610488891601562, + "objective/non_score_reward": -0.680524468421936, + "objective/rlhf_reward": 8.858537673950195, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0009341458207927644, + "policy/clipfrac_avg": 0.006956940051168203, + "policy/entropy_avg": 0.18271827697753906, + "step": 217, + "val/clipfrac_avg": 0.10816793888807297, + "val/num_eos_tokens": 17, + "val/ratio": 0.9906838536262512, + "val/ratio_var": 9.380819392390549e-05 + }, + { + "episode": 3488, + "epoch": 0.17616161616161616, + "eps": 0, + "loss/policy_avg": 0.2500600814819336, + "loss/value_avg": 0.9119748473167419, + "lr": 9.577453530696993e-06, + "objective/entropy": -89.79592895507812, + "objective/kl": 14.33843994140625, + "objective/non_score_reward": -0.7169219851493835, + "objective/rlhf_reward": 9.439327239990234, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0010062719229608774, + "policy/clipfrac_avg": 0.016506358981132507, + "policy/entropy_avg": 0.17433929443359375, + "step": 218, + "val/clipfrac_avg": 0.07429616153240204, + "val/num_eos_tokens": 11, + "val/ratio": 0.9891164302825928, + "val/ratio_var": 4.1760282329050824e-05 + }, + { + "episode": 3504, + "epoch": 0.17696969696969697, + "eps": 0, + "loss/policy_avg": -0.24181823432445526, + "loss/value_avg": 0.22863328456878662, + "lr": 9.572063115079063e-06, + "objective/entropy": -125.28169250488281, + "objective/kl": 12.968160629272461, + "objective/non_score_reward": -0.648408055305481, + "objective/rlhf_reward": 7.792998313903809, + "objective/scores": 8.4375, + "policy/approxkl_avg": 0.0007275404059328139, + "policy/clipfrac_avg": 0.00535131199285388, + "policy/entropy_avg": 0.14211547374725342, + "step": 219, + "val/clipfrac_avg": 0.17199304699897766, + "val/num_eos_tokens": 123, + "val/ratio": 0.9909255504608154, + "val/ratio_var": 6.174561713123694e-05 + }, + { + "episode": 3520, + "epoch": 0.17777777777777778, + "eps": 0, + "loss/policy_avg": -0.12509484589099884, + "loss/value_avg": 0.6027560234069824, + "lr": 9.566640070964756e-06, + "objective/entropy": -49.967041015625, + "objective/kl": 9.079083442687988, + "objective/non_score_reward": -0.45395419001579285, + "objective/rlhf_reward": 10.452295303344727, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008702286286279559, + "policy/clipfrac_avg": 0.006548616103827953, + "policy/entropy_avg": 0.16074752807617188, + "step": 220, + "val/clipfrac_avg": 0.08971220999956131, + "val/num_eos_tokens": 193, + "val/ratio": 0.9901782274246216, + "val/ratio_var": 4.4974971388000995e-05 + }, + { + "episode": 3536, + "epoch": 0.1785858585858586, + "eps": 0, + "loss/policy_avg": -0.00034146755933761597, + "loss/value_avg": 0.4031302332878113, + "lr": 9.561184437055586e-06, + "objective/entropy": -29.953493118286133, + "objective/kl": 15.149253845214844, + "objective/non_score_reward": -0.757462739944458, + "objective/rlhf_reward": 8.062849998474121, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010021317284554243, + "policy/clipfrac_avg": 0.005520673934370279, + "policy/entropy_avg": 0.19618558883666992, + "step": 221, + "val/clipfrac_avg": 0.09643255174160004, + "val/num_eos_tokens": 200, + "val/ratio": 0.9874874353408813, + "val/ratio_var": 3.06899928546045e-05 + }, + { + "episode": 3552, + "epoch": 0.17939393939393938, + "eps": 0, + "loss/policy_avg": -0.0959365963935852, + "loss/value_avg": 2.4835777282714844, + "lr": 9.555696252285648e-06, + "objective/entropy": -90.66460418701172, + "objective/kl": 9.775554656982422, + "objective/non_score_reward": -0.488777756690979, + "objective/rlhf_reward": 11.565910339355469, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000993978464975953, + "policy/clipfrac_avg": 0.007861049845814705, + "policy/entropy_avg": 0.187469482421875, + "step": 222, + "val/clipfrac_avg": 0.22381994128227234, + "val/num_eos_tokens": 27, + "val/ratio": 0.9890456795692444, + "val/ratio_var": 6.582106288988143e-05 + }, + { + "episode": 3568, + "epoch": 0.1802020202020202, + "eps": 0, + "loss/policy_avg": -0.23206055164337158, + "loss/value_avg": 0.20906797051429749, + "lr": 9.550175555821333e-06, + "objective/entropy": -154.86912536621094, + "objective/kl": 17.497539520263672, + "objective/non_score_reward": -0.8748769760131836, + "objective/rlhf_reward": 8.824341773986816, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008605552138760686, + "policy/clipfrac_avg": 0.005713596940040588, + "policy/entropy_avg": 0.17575836181640625, + "step": 223, + "val/clipfrac_avg": 0.09188983589410782, + "val/num_eos_tokens": 225, + "val/ratio": 0.9883838891983032, + "val/ratio_var": 6.594192382181063e-05 + }, + { + "episode": 3584, + "epoch": 0.181010101010101, + "eps": 0, + "loss/policy_avg": -0.0811835527420044, + "loss/value_avg": 1.0116069316864014, + "lr": 9.544622387061055e-06, + "objective/entropy": -123.72830963134766, + "objective/kl": 12.197138786315918, + "objective/non_score_reward": -0.6098569631576538, + "objective/rlhf_reward": 9.116704940795898, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0009076971909962595, + "policy/clipfrac_avg": 0.022357013076543808, + "policy/entropy_avg": 0.19714951515197754, + "step": 224, + "val/clipfrac_avg": 0.12505848705768585, + "val/num_eos_tokens": 310, + "val/ratio": 0.989097535610199, + "val/ratio_var": 5.287026215228252e-05 + }, + { + "episode": 3600, + "epoch": 0.18181818181818182, + "eps": 0, + "loss/policy_avg": 0.032674357295036316, + "loss/value_avg": 1.1174589395523071, + "lr": 9.539036785634961e-06, + "objective/entropy": -127.17562866210938, + "objective/kl": 13.420661926269531, + "objective/non_score_reward": -0.6710331439971924, + "objective/rlhf_reward": 8.625842094421387, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007886404637247324, + "policy/clipfrac_avg": 0.006756953429430723, + "policy/entropy_avg": 0.1559467315673828, + "step": 225, + "val/clipfrac_avg": 0.13264839351177216, + "val/num_eos_tokens": 510, + "val/ratio": 0.9896252155303955, + "val/ratio_var": 5.247633816907182e-05 + }, + { + "episode": 3616, + "epoch": 0.18262626262626264, + "eps": 0, + "loss/policy_avg": -0.2772698402404785, + "loss/value_avg": 0.6462498307228088, + "lr": 9.533418791404658e-06, + "objective/entropy": -67.11093139648438, + "objective/kl": 18.100746154785156, + "objective/non_score_reward": -0.9050373435020447, + "objective/rlhf_reward": 8.509025573730469, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.001084581483155489, + "policy/clipfrac_avg": 0.008873453363776207, + "policy/entropy_avg": 0.17022252082824707, + "step": 226, + "val/clipfrac_avg": 0.08884630352258682, + "val/num_eos_tokens": 123, + "val/ratio": 0.9877827167510986, + "val/ratio_var": 4.777583308168687e-05 + }, + { + "episode": 3632, + "epoch": 0.18343434343434342, + "eps": 0, + "loss/policy_avg": 0.1585109829902649, + "loss/value_avg": 0.5168319940567017, + "lr": 9.527768444462922e-06, + "objective/entropy": -100.66576385498047, + "objective/kl": 15.785117149353027, + "objective/non_score_reward": -0.7892558574676514, + "objective/rlhf_reward": 8.710744857788086, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0010630425531417131, + "policy/clipfrac_avg": 0.010223347693681717, + "policy/entropy_avg": 0.199601411819458, + "step": 227, + "val/clipfrac_avg": 0.06462043523788452, + "val/num_eos_tokens": 393, + "val/ratio": 0.9871734380722046, + "val/ratio_var": 6.318103987723589e-05 + }, + { + "episode": 3648, + "epoch": 0.18424242424242424, + "eps": 0, + "loss/policy_avg": -0.31773215532302856, + "loss/value_avg": 0.8459819555282593, + "lr": 9.522085785133415e-06, + "objective/entropy": -114.29866790771484, + "objective/kl": 12.633371353149414, + "objective/non_score_reward": -0.6316685676574707, + "objective/rlhf_reward": 8.786300659179688, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008345604874193668, + "policy/clipfrac_avg": 0.004700110759586096, + "policy/entropy_avg": 0.16971874237060547, + "step": 228, + "val/clipfrac_avg": 0.126671701669693, + "val/num_eos_tokens": 116, + "val/ratio": 0.9897171258926392, + "val/ratio_var": 5.562942897086032e-05 + }, + { + "episode": 3664, + "epoch": 0.18505050505050505, + "eps": 0, + "loss/policy_avg": 0.04545650631189346, + "loss/value_avg": 1.421910285949707, + "lr": 9.516370853970395e-06, + "objective/entropy": -71.8100814819336, + "objective/kl": 12.768682479858398, + "objective/non_score_reward": -0.638434112071991, + "objective/rlhf_reward": 8.955315589904785, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.0008987415349110961, + "policy/clipfrac_avg": 0.010495506227016449, + "policy/entropy_avg": 0.17481613159179688, + "step": 229, + "val/clipfrac_avg": 0.17400485277175903, + "val/num_eos_tokens": 326, + "val/ratio": 0.9892676472663879, + "val/ratio_var": 7.376643043244258e-05 + }, + { + "episode": 3680, + "epoch": 0.18585858585858586, + "eps": 0, + "loss/policy_avg": 0.03276506066322327, + "loss/value_avg": 0.5139911770820618, + "lr": 9.510623691758428e-06, + "objective/entropy": -140.628662109375, + "objective/kl": 13.719542503356934, + "objective/non_score_reward": -0.6859771013259888, + "objective/rlhf_reward": 8.6304292678833, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007748354692012072, + "policy/clipfrac_avg": 0.008184252306818962, + "policy/entropy_avg": 0.1518726348876953, + "step": 230, + "val/clipfrac_avg": 0.15990711748600006, + "val/num_eos_tokens": 475, + "val/ratio": 0.9908199310302734, + "val/ratio_var": 4.362429535831325e-05 + }, + { + "episode": 3696, + "epoch": 0.18666666666666668, + "eps": 0, + "loss/policy_avg": 0.3543701469898224, + "loss/value_avg": 0.7293742895126343, + "lr": 9.504844339512096e-06, + "objective/entropy": -112.58895874023438, + "objective/kl": 13.595596313476562, + "objective/non_score_reward": -0.6797798871994019, + "objective/rlhf_reward": 6.999907493591309, + "objective/scores": 7.6875, + "policy/approxkl_avg": 0.0007951147854328156, + "policy/clipfrac_avg": 0.014235245995223522, + "policy/entropy_avg": 0.13730275630950928, + "step": 231, + "val/clipfrac_avg": 0.14041440188884735, + "val/num_eos_tokens": 31, + "val/ratio": 0.9905531406402588, + "val/ratio_var": 4.2730218410724774e-05 + }, + { + "episode": 3712, + "epoch": 0.1874747474747475, + "eps": 0, + "loss/policy_avg": 0.07381056249141693, + "loss/value_avg": 0.374658465385437, + "lr": 9.499032838475708e-06, + "objective/entropy": -95.20710754394531, + "objective/kl": 15.552490234375, + "objective/non_score_reward": -0.7776244878768921, + "objective/rlhf_reward": 8.195032119750977, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009295077761635184, + "policy/clipfrac_avg": 0.0064463308081030846, + "policy/entropy_avg": 0.17013072967529297, + "step": 232, + "val/clipfrac_avg": 0.08955052495002747, + "val/num_eos_tokens": 338, + "val/ratio": 0.9885688424110413, + "val/ratio_var": 4.924907989334315e-05 + }, + { + "episode": 3728, + "epoch": 0.18828282828282827, + "eps": 0, + "loss/policy_avg": -0.3743169605731964, + "loss/value_avg": 1.4489904642105103, + "lr": 9.493189230123e-06, + "objective/entropy": -55.171142578125, + "objective/kl": 11.425302505493164, + "objective/non_score_reward": -0.5712651014328003, + "objective/rlhf_reward": 11.71779727935791, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009924890473484993, + "policy/clipfrac_avg": 0.015823353081941605, + "policy/entropy_avg": 0.15912246704101562, + "step": 233, + "val/clipfrac_avg": 0.3463912606239319, + "val/num_eos_tokens": 212, + "val/ratio": 0.9904276132583618, + "val/ratio_var": 4.2650059185689315e-05 + }, + { + "episode": 3744, + "epoch": 0.1890909090909091, + "eps": 0, + "loss/policy_avg": 0.1211954802274704, + "loss/value_avg": 1.033839225769043, + "lr": 9.48731355615684e-06, + "objective/entropy": -114.34797668457031, + "objective/kl": 11.218280792236328, + "objective/non_score_reward": -0.5609140396118164, + "objective/rlhf_reward": 6.798460960388184, + "objective/scores": 7.375, + "policy/approxkl_avg": 0.0013848431408405304, + "policy/clipfrac_avg": 0.013197123073041439, + "policy/entropy_avg": 0.1911311149597168, + "step": 234, + "val/clipfrac_avg": 0.2574499845504761, + "val/num_eos_tokens": 162, + "val/ratio": 0.9891800284385681, + "val/ratio_var": 6.6911663452629e-05 + }, + { + "episode": 3760, + "epoch": 0.1898989898989899, + "eps": 0, + "loss/policy_avg": -0.05456429719924927, + "loss/value_avg": 2.0512852668762207, + "lr": 9.481405858508935e-06, + "objective/entropy": -180.02139282226562, + "objective/kl": 14.93826675415039, + "objective/non_score_reward": -0.7469133138656616, + "objective/rlhf_reward": 8.26871109008789, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0006930871168151498, + "policy/clipfrac_avg": 0.0112287737429142, + "policy/entropy_avg": 0.12185859680175781, + "step": 235, + "val/clipfrac_avg": 0.16021409630775452, + "val/num_eos_tokens": 303, + "val/ratio": 0.9917303323745728, + "val/ratio_var": 3.9941103750607e-05 + }, + { + "episode": 3776, + "epoch": 0.1907070707070707, + "eps": 0, + "loss/policy_avg": -0.27526289224624634, + "loss/value_avg": 0.6880919933319092, + "lr": 9.475466179339528e-06, + "objective/entropy": -145.46041870117188, + "objective/kl": 14.790599822998047, + "objective/non_score_reward": -0.7395299673080444, + "objective/rlhf_reward": 10.033906936645508, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0008720404002815485, + "policy/clipfrac_avg": 0.005699498578906059, + "policy/entropy_avg": 0.1630878448486328, + "step": 236, + "val/clipfrac_avg": 0.061457473784685135, + "val/num_eos_tokens": 592, + "val/ratio": 0.9893093109130859, + "val/ratio_var": 5.2708415751112625e-05 + }, + { + "episode": 3792, + "epoch": 0.19151515151515153, + "eps": 0, + "loss/policy_avg": 0.15451845526695251, + "loss/value_avg": 1.8476499319076538, + "lr": 9.469494561037097e-06, + "objective/entropy": -107.19807434082031, + "objective/kl": 14.95478630065918, + "objective/non_score_reward": -0.747739315032959, + "objective/rlhf_reward": 8.982728958129883, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0011050915345549583, + "policy/clipfrac_avg": 0.032157786190509796, + "policy/entropy_avg": 0.18476414680480957, + "step": 237, + "val/clipfrac_avg": 0.08973570913076401, + "val/num_eos_tokens": 680, + "val/ratio": 0.9881956577301025, + "val/ratio_var": 5.36524603376165e-05 + }, + { + "episode": 3808, + "epoch": 0.1923232323232323, + "eps": 0, + "loss/policy_avg": -0.007887881249189377, + "loss/value_avg": 0.22297966480255127, + "lr": 9.463491046218058e-06, + "objective/entropy": -146.70474243164062, + "objective/kl": 15.001608848571777, + "objective/non_score_reward": -0.7500803470611572, + "objective/rlhf_reward": 7.753825664520264, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0008398975478485227, + "policy/clipfrac_avg": 0.005974689032882452, + "policy/entropy_avg": 0.15901756286621094, + "step": 238, + "val/clipfrac_avg": 0.08070166409015656, + "val/num_eos_tokens": 343, + "val/ratio": 0.9894462823867798, + "val/ratio_var": 6.362869316944852e-05 + }, + { + "episode": 3824, + "epoch": 0.19313131313131313, + "eps": 0, + "loss/policy_avg": 0.06090904772281647, + "loss/value_avg": 0.8647578358650208, + "lr": 9.457455677726447e-06, + "objective/entropy": -49.81889343261719, + "objective/kl": 11.3671875, + "objective/non_score_reward": -0.568359375, + "objective/rlhf_reward": 9.869140625, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0011585194151848555, + "policy/clipfrac_avg": 0.00720559898763895, + "policy/entropy_avg": 0.18108272552490234, + "step": 239, + "val/clipfrac_avg": 0.1332775205373764, + "val/num_eos_tokens": 15, + "val/ratio": 0.9874423146247864, + "val/ratio_var": 7.534508768003434e-05 + }, + { + "episode": 3840, + "epoch": 0.19393939393939394, + "eps": 0, + "loss/policy_avg": -0.0028132200241088867, + "loss/value_avg": 0.47351714968681335, + "lr": 9.451388498633635e-06, + "objective/entropy": -92.73252868652344, + "objective/kl": 13.683374404907227, + "objective/non_score_reward": -0.6841686964035034, + "objective/rlhf_reward": 9.847081184387207, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000987341278232634, + "policy/clipfrac_avg": 0.011263278312981129, + "policy/entropy_avg": 0.1565999984741211, + "step": 240, + "val/clipfrac_avg": 0.12548519670963287, + "val/num_eos_tokens": 335, + "val/ratio": 0.9901664853096008, + "val/ratio_var": 5.156149563845247e-05 + }, + { + "episode": 3856, + "epoch": 0.19474747474747475, + "eps": 0, + "loss/policy_avg": -0.11238707602024078, + "loss/value_avg": 0.30569231510162354, + "lr": 9.445289552237998e-06, + "objective/entropy": -94.36567687988281, + "objective/kl": 14.898351669311523, + "objective/non_score_reward": -0.744917631149292, + "objective/rlhf_reward": 9.731644630432129, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008329006377607584, + "policy/clipfrac_avg": 0.005581840872764587, + "policy/entropy_avg": 0.16306018829345703, + "step": 241, + "val/clipfrac_avg": 0.14762382209300995, + "val/num_eos_tokens": 797, + "val/ratio": 0.9892506003379822, + "val/ratio_var": 4.0011495002545416e-05 + }, + { + "episode": 3872, + "epoch": 0.19555555555555557, + "eps": 0, + "loss/policy_avg": 0.11825011670589447, + "loss/value_avg": 0.4140797257423401, + "lr": 9.439158882064623e-06, + "objective/entropy": -108.59629821777344, + "objective/kl": 13.85297966003418, + "objective/non_score_reward": -0.6926490068435669, + "objective/rlhf_reward": 9.354225158691406, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008656422724016011, + "policy/clipfrac_avg": 0.008920292370021343, + "policy/entropy_avg": 0.1616191864013672, + "step": 242, + "val/clipfrac_avg": 0.2358715534210205, + "val/num_eos_tokens": 95, + "val/ratio": 0.9895873069763184, + "val/ratio_var": 5.987072654534131e-05 + }, + { + "episode": 3888, + "epoch": 0.19636363636363635, + "eps": 0, + "loss/policy_avg": -0.12114518135786057, + "loss/value_avg": 0.5617895126342773, + "lr": 9.432996531865001e-06, + "objective/entropy": -158.32139587402344, + "objective/kl": 20.37476348876953, + "objective/non_score_reward": -1.0187382698059082, + "objective/rlhf_reward": 7.76641845703125, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008469723397865891, + "policy/clipfrac_avg": 0.017248257994651794, + "policy/entropy_avg": 0.17039668560028076, + "step": 243, + "val/clipfrac_avg": 0.18548327684402466, + "val/num_eos_tokens": 629, + "val/ratio": 0.9896106719970703, + "val/ratio_var": 5.761132706538774e-05 + }, + { + "episode": 3904, + "epoch": 0.19717171717171716, + "eps": 0, + "loss/policy_avg": 0.11465314775705338, + "loss/value_avg": 0.3883507251739502, + "lr": 9.426802545616696e-06, + "objective/entropy": -76.740478515625, + "objective/kl": 13.425769805908203, + "objective/non_score_reward": -0.6712884902954102, + "objective/rlhf_reward": 8.80527400970459, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0008673456613905728, + "policy/clipfrac_avg": 0.010126919485628605, + "policy/entropy_avg": 0.14852988719940186, + "step": 244, + "val/clipfrac_avg": 0.08817487955093384, + "val/num_eos_tokens": 105, + "val/ratio": 0.9893420934677124, + "val/ratio_var": 4.652348070521839e-05 + }, + { + "episode": 3920, + "epoch": 0.19797979797979798, + "eps": 0, + "loss/policy_avg": 0.07142084836959839, + "loss/value_avg": 0.4675530195236206, + "lr": 9.420576967523049e-06, + "objective/entropy": -103.8277587890625, + "objective/kl": 14.990248680114746, + "objective/non_score_reward": -0.7495124340057373, + "objective/rlhf_reward": 7.506346702575684, + "objective/scores": 8.25, + "policy/approxkl_avg": 0.0008764934027567506, + "policy/clipfrac_avg": 0.010414833202958107, + "policy/entropy_avg": 0.17772126197814941, + "step": 245, + "val/clipfrac_avg": 0.16199971735477448, + "val/num_eos_tokens": 463, + "val/ratio": 0.9890564680099487, + "val/ratio_var": 5.0618149543879554e-05 + }, + { + "episode": 3936, + "epoch": 0.1987878787878788, + "eps": 0, + "loss/policy_avg": 0.09155968576669693, + "loss/value_avg": 1.220616102218628, + "lr": 9.414319842012855e-06, + "objective/entropy": -68.2171401977539, + "objective/kl": 15.050432205200195, + "objective/non_score_reward": -0.7525216341018677, + "objective/rlhf_reward": 8.286540985107422, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0013208398595452309, + "policy/clipfrac_avg": 0.017676567658782005, + "policy/entropy_avg": 0.2582554817199707, + "step": 246, + "val/clipfrac_avg": 0.12387647479772568, + "val/num_eos_tokens": 183, + "val/ratio": 0.9864345192909241, + "val/ratio_var": 7.634109351783991e-05 + }, + { + "episode": 3952, + "epoch": 0.1995959595959596, + "eps": 0, + "loss/policy_avg": 0.10812592506408691, + "loss/value_avg": 0.3838285803794861, + "lr": 9.408031213740045e-06, + "objective/entropy": -98.11519622802734, + "objective/kl": 12.254688262939453, + "objective/non_score_reward": -0.6127344369888306, + "objective/rlhf_reward": 7.926328182220459, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.000676511088386178, + "policy/clipfrac_avg": 0.004696001298725605, + "policy/entropy_avg": 0.13373851776123047, + "step": 247, + "val/clipfrac_avg": 0.27838826179504395, + "val/num_eos_tokens": 74, + "val/ratio": 0.9912145137786865, + "val/ratio_var": 3.239799843868241e-05 + }, + { + "episode": 3968, + "epoch": 0.20040404040404042, + "eps": 0, + "loss/policy_avg": 0.2526165246963501, + "loss/value_avg": 0.49756327271461487, + "lr": 9.401711127583372e-06, + "objective/entropy": -93.28247833251953, + "objective/kl": 14.079293251037598, + "objective/non_score_reward": -0.7039647102355957, + "objective/rlhf_reward": 8.092909812927246, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010151651222258806, + "policy/clipfrac_avg": 0.010710397735238075, + "policy/entropy_avg": 0.17333316802978516, + "step": 248, + "val/clipfrac_avg": 0.0955374613404274, + "val/num_eos_tokens": 329, + "val/ratio": 0.9887602925300598, + "val/ratio_var": 6.146164378151298e-05 + }, + { + "episode": 3984, + "epoch": 0.2012121212121212, + "eps": 0, + "loss/policy_avg": -0.3386799097061157, + "loss/value_avg": 0.6449569463729858, + "lr": 9.395359628646087e-06, + "objective/entropy": -106.61834716796875, + "objective/kl": 15.157907485961914, + "objective/non_score_reward": -0.7578953504562378, + "objective/rlhf_reward": 9.171792984008789, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008972659707069397, + "policy/clipfrac_avg": 0.017500415444374084, + "policy/entropy_avg": 0.1698697805404663, + "step": 249, + "val/clipfrac_avg": 0.16914868354797363, + "val/num_eos_tokens": 107, + "val/ratio": 0.9900318384170532, + "val/ratio_var": 6.466583727160469e-05 + }, + { + "episode": 4000, + "epoch": 0.20202020202020202, + "eps": 0, + "loss/policy_avg": -0.26445555686950684, + "loss/value_avg": 1.824251413345337, + "lr": 9.388976762255615e-06, + "objective/entropy": -88.12916564941406, + "objective/kl": 15.847978591918945, + "objective/non_score_reward": -0.7923988699913025, + "objective/rlhf_reward": 9.281820297241211, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008180903387255967, + "policy/clipfrac_avg": 0.0073817213997244835, + "policy/entropy_avg": 0.15668296813964844, + "step": 250, + "val/clipfrac_avg": 0.09538544714450836, + "val/num_eos_tokens": 460, + "val/ratio": 0.9894523620605469, + "val/ratio_var": 4.9866332119563594e-05 + }, + { + "episode": 4016, + "epoch": 0.20282828282828283, + "eps": 0, + "loss/policy_avg": -0.03636375069618225, + "loss/value_avg": 0.24154099822044373, + "lr": 9.382562573963238e-06, + "objective/entropy": -53.91804122924805, + "objective/kl": 15.981526374816895, + "objective/non_score_reward": -0.7990763187408447, + "objective/rlhf_reward": 7.157954692840576, + "objective/scores": 7.96875, + "policy/approxkl_avg": 0.0009486594935879111, + "policy/clipfrac_avg": 0.008417891338467598, + "policy/entropy_avg": 0.1822967529296875, + "step": 251, + "val/clipfrac_avg": 0.13178351521492004, + "val/num_eos_tokens": 305, + "val/ratio": 0.9880446195602417, + "val/ratio_var": 4.493531378102489e-05 + }, + { + "episode": 4032, + "epoch": 0.20363636363636364, + "eps": 0, + "loss/policy_avg": -0.04389417916536331, + "loss/value_avg": 0.5461421012878418, + "lr": 9.376117109543769e-06, + "objective/entropy": -116.62461853027344, + "objective/kl": 8.785242080688477, + "objective/non_score_reward": -0.43926215171813965, + "objective/rlhf_reward": 9.603706359863281, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0005816442426294088, + "policy/clipfrac_avg": 0.004240636248141527, + "policy/entropy_avg": 0.15207290649414062, + "step": 252, + "val/clipfrac_avg": 0.24988536536693573, + "val/num_eos_tokens": 497, + "val/ratio": 0.9932804703712463, + "val/ratio_var": 4.206900121062063e-05 + }, + { + "episode": 4048, + "epoch": 0.20444444444444446, + "eps": 0, + "loss/policy_avg": 0.3184441924095154, + "loss/value_avg": 0.8039752244949341, + "lr": 9.369640414995216e-06, + "objective/entropy": -88.22281646728516, + "objective/kl": 11.825658798217773, + "objective/non_score_reward": -0.5912830233573914, + "objective/rlhf_reward": 8.143091201782227, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008585604373365641, + "policy/clipfrac_avg": 0.015919335186481476, + "policy/entropy_avg": 0.1344587802886963, + "step": 253, + "val/clipfrac_avg": 0.05827297270298004, + "val/num_eos_tokens": 746, + "val/ratio": 0.9898501634597778, + "val/ratio_var": 6.264803232625127e-05 + }, + { + "episode": 4064, + "epoch": 0.20525252525252524, + "eps": 0, + "loss/policy_avg": 0.06932681798934937, + "loss/value_avg": 0.4704505503177643, + "lr": 9.363132536538466e-06, + "objective/entropy": -42.02299880981445, + "objective/kl": 14.882261276245117, + "objective/non_score_reward": -0.744113028049469, + "objective/rlhf_reward": 8.107449531555176, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0009829355403780937, + "policy/clipfrac_avg": 0.019861944019794464, + "policy/entropy_avg": 0.18294620513916016, + "step": 254, + "val/clipfrac_avg": 0.09604226052761078, + "val/num_eos_tokens": 202, + "val/ratio": 0.9877536296844482, + "val/ratio_var": 5.442448673420586e-05 + }, + { + "episode": 4080, + "epoch": 0.20606060606060606, + "eps": 0, + "loss/policy_avg": 0.1454492211341858, + "loss/value_avg": 0.2512233257293701, + "lr": 9.356593520616948e-06, + "objective/entropy": -132.06040954589844, + "objective/kl": 15.13500690460205, + "objective/non_score_reward": -0.7567504048347473, + "objective/rlhf_reward": 8.192468643188477, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0007790173403918743, + "policy/clipfrac_avg": 0.019181344658136368, + "policy/entropy_avg": 0.13058090209960938, + "step": 255, + "val/clipfrac_avg": 0.0804247111082077, + "val/num_eos_tokens": 424, + "val/ratio": 0.9907982349395752, + "val/ratio_var": 3.778808968490921e-05 + }, + { + "episode": 4096, + "epoch": 0.20686868686868687, + "eps": 0, + "loss/policy_avg": -0.1870514154434204, + "loss/value_avg": 0.8562079668045044, + "lr": 9.350023413896302e-06, + "objective/entropy": -94.291748046875, + "objective/kl": 17.12119483947754, + "objective/non_score_reward": -0.8560597896575928, + "objective/rlhf_reward": 8.671283721923828, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009006272302940488, + "policy/clipfrac_avg": 0.005755542777478695, + "policy/entropy_avg": 0.1746842861175537, + "step": 256, + "val/clipfrac_avg": 0.09692853689193726, + "val/num_eos_tokens": 798, + "val/ratio": 0.9885848164558411, + "val/ratio_var": 4.2367017158539966e-05 + }, + { + "episode": 4112, + "epoch": 0.20767676767676768, + "eps": 0, + "loss/policy_avg": -0.6601881384849548, + "loss/value_avg": 1.8542723655700684, + "lr": 9.34342226326405e-06, + "objective/entropy": -157.23556518554688, + "objective/kl": 19.609127044677734, + "objective/non_score_reward": -0.9804563522338867, + "objective/rlhf_reward": 7.660168170928955, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0007378142327070236, + "policy/clipfrac_avg": 0.009470398537814617, + "policy/entropy_avg": 0.17036652565002441, + "step": 257, + "val/clipfrac_avg": 0.17326012253761292, + "val/num_eos_tokens": 383, + "val/ratio": 0.990649938583374, + "val/ratio_var": 5.1252005505375564e-05 + }, + { + "episode": 4128, + "epoch": 0.2084848484848485, + "eps": 0, + "loss/policy_avg": 0.0006968677043914795, + "loss/value_avg": 0.5727497935295105, + "lr": 9.336790115829255e-06, + "objective/entropy": -95.90520477294922, + "objective/kl": 11.775390625, + "objective/non_score_reward": -0.5887695550918579, + "objective/rlhf_reward": 7.544043064117432, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007710008067078888, + "policy/clipfrac_avg": 0.008710110560059547, + "policy/entropy_avg": 0.13333511352539062, + "step": 258, + "val/clipfrac_avg": 0.20523446798324585, + "val/num_eos_tokens": 553, + "val/ratio": 0.9907342195510864, + "val/ratio_var": 5.306022649165243e-05 + }, + { + "episode": 4144, + "epoch": 0.20929292929292928, + "eps": 0, + "loss/policy_avg": -0.11990015208721161, + "loss/value_avg": 0.18013015389442444, + "lr": 9.330127018922195e-06, + "objective/entropy": -180.310791015625, + "objective/kl": 15.882448196411133, + "objective/non_score_reward": -0.7941223382949829, + "objective/rlhf_reward": 7.748846530914307, + "objective/scores": 8.5625, + "policy/approxkl_avg": 0.0007087202975526452, + "policy/clipfrac_avg": 0.008986853063106537, + "policy/entropy_avg": 0.14745712280273438, + "step": 259, + "val/clipfrac_avg": 0.14693328738212585, + "val/num_eos_tokens": 400, + "val/ratio": 0.9911876916885376, + "val/ratio_var": 5.386102566262707e-05 + }, + { + "episode": 4160, + "epoch": 0.2101010101010101, + "eps": 0, + "loss/policy_avg": 0.046490609645843506, + "loss/value_avg": 1.0619938373565674, + "lr": 9.32343302009401e-06, + "objective/entropy": -97.93316650390625, + "objective/kl": 11.504565238952637, + "objective/non_score_reward": -0.5752282738685608, + "objective/rlhf_reward": 10.410123825073242, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.001048518344759941, + "policy/clipfrac_avg": 0.022185876965522766, + "policy/entropy_avg": 0.16389119625091553, + "step": 260, + "val/clipfrac_avg": 0.2877679169178009, + "val/num_eos_tokens": 62, + "val/ratio": 0.989488959312439, + "val/ratio_var": 7.833181007299572e-05 + }, + { + "episode": 4176, + "epoch": 0.2109090909090909, + "eps": 0, + "loss/policy_avg": -0.20035383105278015, + "loss/value_avg": 0.5576207041740417, + "lr": 9.316708167116377e-06, + "objective/entropy": -75.23338317871094, + "objective/kl": 14.002493858337402, + "objective/non_score_reward": -0.7001247406005859, + "objective/rlhf_reward": 10.026437759399414, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009849893394857645, + "policy/clipfrac_avg": 0.006675241515040398, + "policy/entropy_avg": 0.17143821716308594, + "step": 261, + "val/clipfrac_avg": 0.16675466299057007, + "val/num_eos_tokens": 11, + "val/ratio": 0.9886298775672913, + "val/ratio_var": 6.11293435213156e-05 + }, + { + "episode": 4192, + "epoch": 0.21171717171717172, + "eps": 0, + "loss/policy_avg": 0.04166821762919426, + "loss/value_avg": 1.0122075080871582, + "lr": 9.309952507981164e-06, + "objective/entropy": -105.865234375, + "objective/kl": 14.501296043395996, + "objective/non_score_reward": -0.7250648140907288, + "objective/rlhf_reward": 10.212434768676758, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0009169665863737464, + "policy/clipfrac_avg": 0.013359840959310532, + "policy/entropy_avg": 0.1592082977294922, + "step": 262, + "val/clipfrac_avg": 0.11749286949634552, + "val/num_eos_tokens": 54, + "val/ratio": 0.9892106056213379, + "val/ratio_var": 5.9744346799561754e-05 + }, + { + "episode": 4208, + "epoch": 0.21252525252525253, + "eps": 0, + "loss/policy_avg": -0.12460523098707199, + "loss/value_avg": 0.3560577630996704, + "lr": 9.303166090900082e-06, + "objective/entropy": -108.7623519897461, + "objective/kl": 12.704278945922852, + "objective/non_score_reward": -0.6352140307426453, + "objective/rlhf_reward": 8.814004898071289, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0007622368866577744, + "policy/clipfrac_avg": 0.012059643864631653, + "policy/entropy_avg": 0.1394658088684082, + "step": 263, + "val/clipfrac_avg": 0.0886392593383789, + "val/num_eos_tokens": 196, + "val/ratio": 0.9914352893829346, + "val/ratio_var": 3.335260771564208e-05 + }, + { + "episode": 4224, + "epoch": 0.21333333333333335, + "eps": 0, + "loss/policy_avg": 0.3000638484954834, + "loss/value_avg": 3.231374502182007, + "lr": 9.296348964304351e-06, + "objective/entropy": -82.82820129394531, + "objective/kl": 11.76550579071045, + "objective/non_score_reward": -0.5882752537727356, + "objective/rlhf_reward": 9.841412544250488, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.000694192829541862, + "policy/clipfrac_avg": 0.007227397058159113, + "policy/entropy_avg": 0.12492609024047852, + "step": 264, + "val/clipfrac_avg": 0.07290898263454437, + "val/num_eos_tokens": 336, + "val/ratio": 0.9919120073318481, + "val/ratio_var": 3.886450213030912e-05 + }, + { + "episode": 4240, + "epoch": 0.21414141414141413, + "eps": 0, + "loss/policy_avg": 0.1362697035074234, + "loss/value_avg": 0.25763940811157227, + "lr": 9.289501176844348e-06, + "objective/entropy": -161.09799194335938, + "objective/kl": 15.946310043334961, + "objective/non_score_reward": -0.7973154783248901, + "objective/rlhf_reward": 8.17143440246582, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.000784806557931006, + "policy/clipfrac_avg": 0.005639348179101944, + "policy/entropy_avg": 0.14366579055786133, + "step": 265, + "val/clipfrac_avg": 0.034823767840862274, + "val/num_eos_tokens": 285, + "val/ratio": 0.9907838702201843, + "val/ratio_var": 4.943497697240673e-05 + }, + { + "episode": 4256, + "epoch": 0.21494949494949495, + "eps": 0, + "loss/policy_avg": -0.14799898862838745, + "loss/value_avg": 1.1736255884170532, + "lr": 9.282622777389258e-06, + "objective/entropy": -38.92912673950195, + "objective/kl": 18.608516693115234, + "objective/non_score_reward": -0.9304258227348328, + "objective/rlhf_reward": 9.499261856079102, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0010509174317121506, + "policy/clipfrac_avg": 0.013766787946224213, + "policy/entropy_avg": 0.17187881469726562, + "step": 266, + "val/clipfrac_avg": 0.16095665097236633, + "val/num_eos_tokens": 39, + "val/ratio": 0.9915172457695007, + "val/ratio_var": 0.00022518265177495778 + }, + { + "episode": 4272, + "epoch": 0.21575757575757576, + "eps": 0, + "loss/policy_avg": -0.10845097154378891, + "loss/value_avg": 1.3597133159637451, + "lr": 9.275713815026732e-06, + "objective/entropy": -55.39912414550781, + "objective/kl": 12.634617805480957, + "objective/non_score_reward": -0.6317309141159058, + "objective/rlhf_reward": 9.555768966674805, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0009083591867238283, + "policy/clipfrac_avg": 0.0047781881876289845, + "policy/entropy_avg": 0.11792755126953125, + "step": 267, + "val/clipfrac_avg": 0.17677703499794006, + "val/num_eos_tokens": 131, + "val/ratio": 0.9929987192153931, + "val/ratio_var": 0.00012306030839681625 + }, + { + "episode": 4288, + "epoch": 0.21656565656565657, + "eps": 0, + "loss/policy_avg": -0.40072306990623474, + "loss/value_avg": 0.6274000406265259, + "lr": 9.268774339062529e-06, + "objective/entropy": -59.651851654052734, + "objective/kl": 15.504014015197754, + "objective/non_score_reward": -0.7752006649971008, + "objective/rlhf_reward": 8.392767906188965, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0009320800891146064, + "policy/clipfrac_avg": 0.00602041557431221, + "policy/entropy_avg": 0.16248130798339844, + "step": 268, + "val/clipfrac_avg": 0.24285002052783966, + "val/num_eos_tokens": 26, + "val/ratio": 0.9889825582504272, + "val/ratio_var": 4.561135210678913e-05 + }, + { + "episode": 4304, + "epoch": 0.21737373737373739, + "eps": 0, + "loss/policy_avg": -0.4726325273513794, + "loss/value_avg": 1.0873943567276, + "lr": 9.261804399020175e-06, + "objective/entropy": -117.23480224609375, + "objective/kl": 17.365135192871094, + "objective/non_score_reward": -0.8682568073272705, + "objective/rlhf_reward": 8.834868431091309, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0008819050854071975, + "policy/clipfrac_avg": 0.015071302652359009, + "policy/entropy_avg": 0.16169071197509766, + "step": 269, + "val/clipfrac_avg": 0.06512845307588577, + "val/num_eos_tokens": 367, + "val/ratio": 0.9895751476287842, + "val/ratio_var": 6.195653259055689e-05 + }, + { + "episode": 4320, + "epoch": 0.21818181818181817, + "eps": 0, + "loss/policy_avg": -0.2973840832710266, + "loss/value_avg": 0.6133067607879639, + "lr": 9.254804044640596e-06, + "objective/entropy": -90.55671691894531, + "objective/kl": 14.76534652709961, + "objective/non_score_reward": -0.7382673025131226, + "objective/rlhf_reward": 8.56641960144043, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0007824086351320148, + "policy/clipfrac_avg": 0.005687411408871412, + "policy/entropy_avg": 0.14926570653915405, + "step": 270, + "val/clipfrac_avg": 0.1254422813653946, + "val/num_eos_tokens": 217, + "val/ratio": 0.9903734922409058, + "val/ratio_var": 4.6835259126964957e-05 + }, + { + "episode": 4336, + "epoch": 0.21898989898989898, + "eps": 0, + "loss/policy_avg": 0.10686540603637695, + "loss/value_avg": 0.35223832726478577, + "lr": 9.24777332588177e-06, + "objective/entropy": -108.28328704833984, + "objective/kl": 15.878630638122559, + "objective/non_score_reward": -0.7939314842224121, + "objective/rlhf_reward": 7.127943515777588, + "objective/scores": 7.9375, + "policy/approxkl_avg": 0.0007746703922748566, + "policy/clipfrac_avg": 0.020203590393066406, + "policy/entropy_avg": 0.1530628204345703, + "step": 271, + "val/clipfrac_avg": 0.11016659438610077, + "val/num_eos_tokens": 448, + "val/ratio": 0.9914586544036865, + "val/ratio_var": 4.422034908202477e-05 + }, + { + "episode": 4352, + "epoch": 0.2197979797979798, + "eps": 0, + "loss/policy_avg": -0.13800494372844696, + "loss/value_avg": 1.3003135919570923, + "lr": 9.240712292918375e-06, + "objective/entropy": -133.5970458984375, + "objective/kl": 16.335315704345703, + "objective/non_score_reward": -0.8167657852172852, + "objective/rlhf_reward": 8.417609214782715, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0006824146257713437, + "policy/clipfrac_avg": 0.003920457791537046, + "policy/entropy_avg": 0.12821245193481445, + "step": 272, + "val/clipfrac_avg": 0.1513872593641281, + "val/num_eos_tokens": 1032, + "val/ratio": 0.9920010566711426, + "val/ratio_var": 4.987249485566281e-05 + }, + { + "episode": 4368, + "epoch": 0.2206060606060606, + "eps": 0, + "loss/policy_avg": -0.41694945096969604, + "loss/value_avg": 1.5838637351989746, + "lr": 9.233620996141421e-06, + "objective/entropy": -69.04071807861328, + "objective/kl": 15.891003608703613, + "objective/non_score_reward": -0.7945501804351807, + "objective/rlhf_reward": 8.232793807983398, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0009435138781554997, + "policy/clipfrac_avg": 0.00690112542361021, + "policy/entropy_avg": 0.17545413970947266, + "step": 273, + "val/clipfrac_avg": 0.15208563208580017, + "val/num_eos_tokens": 95, + "val/ratio": 0.9881666898727417, + "val/ratio_var": 9.140658949036151e-05 + }, + { + "episode": 4384, + "epoch": 0.22141414141414142, + "eps": 0, + "loss/policy_avg": -0.23135730624198914, + "loss/value_avg": 1.036683440208435, + "lr": 9.2264994861579e-06, + "objective/entropy": -30.563913345336914, + "objective/kl": 14.84071159362793, + "objective/non_score_reward": -0.7420356273651123, + "objective/rlhf_reward": 9.750152587890625, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008996215765364468, + "policy/clipfrac_avg": 0.0041840290650725365, + "policy/entropy_avg": 0.16372394561767578, + "step": 274, + "val/clipfrac_avg": 0.09715648740530014, + "val/num_eos_tokens": 201, + "val/ratio": 0.9890379905700684, + "val/ratio_var": 5.0841495976783335e-05 + }, + { + "episode": 4400, + "epoch": 0.2222222222222222, + "eps": 0, + "loss/policy_avg": -0.008091986179351807, + "loss/value_avg": 0.7564100623130798, + "lr": 9.219347813790416e-06, + "objective/entropy": -208.7753448486328, + "objective/kl": 19.293121337890625, + "objective/non_score_reward": -0.9646561145782471, + "objective/rlhf_reward": 8.429875373840332, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.00103966542519629, + "policy/clipfrac_avg": 0.021843386813998222, + "policy/entropy_avg": 0.13889122009277344, + "step": 275, + "val/clipfrac_avg": 0.11226043850183487, + "val/num_eos_tokens": 316, + "val/ratio": 0.9908844232559204, + "val/ratio_var": 4.5354645408224314e-05 + }, + { + "episode": 4416, + "epoch": 0.22303030303030302, + "eps": 0, + "loss/policy_avg": -0.012354940176010132, + "loss/value_avg": 1.0118482112884521, + "lr": 9.212166030076832e-06, + "objective/entropy": -111.80735778808594, + "objective/kl": 12.331647872924805, + "objective/non_score_reward": -0.6165823936462402, + "objective/rlhf_reward": 11.742793083190918, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008045949507504702, + "policy/clipfrac_avg": 0.008964905515313148, + "policy/entropy_avg": 0.14202404022216797, + "step": 276, + "val/clipfrac_avg": 0.18904702365398407, + "val/num_eos_tokens": 618, + "val/ratio": 0.9908614754676819, + "val/ratio_var": 6.561782356584445e-05 + }, + { + "episode": 4432, + "epoch": 0.22383838383838384, + "eps": 0, + "loss/policy_avg": -0.44561365246772766, + "loss/value_avg": 4.015078067779541, + "lr": 9.204954186269894e-06, + "objective/entropy": -201.30746459960938, + "objective/kl": 16.085712432861328, + "objective/non_score_reward": -0.8042856454849243, + "objective/rlhf_reward": 10.590245246887207, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000744504330214113, + "policy/clipfrac_avg": 0.010385589674115181, + "policy/entropy_avg": 0.1358182281255722, + "step": 277, + "val/clipfrac_avg": 0.10327339172363281, + "val/num_eos_tokens": 695, + "val/ratio": 0.9907582998275757, + "val/ratio_var": 6.718975055264309e-05 + }, + { + "episode": 4448, + "epoch": 0.22464646464646465, + "eps": 0, + "loss/policy_avg": -0.08938002586364746, + "loss/value_avg": 0.308234304189682, + "lr": 9.197712333836875e-06, + "objective/entropy": -81.09490966796875, + "objective/kl": 19.134828567504883, + "objective/non_score_reward": -0.956741452217102, + "objective/rlhf_reward": 8.371383666992188, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009207304683513939, + "policy/clipfrac_avg": 0.007468705996870995, + "policy/entropy_avg": 0.16699981689453125, + "step": 278, + "val/clipfrac_avg": 0.11933809518814087, + "val/num_eos_tokens": 70, + "val/ratio": 0.9889715909957886, + "val/ratio_var": 3.133176142000593e-05 + }, + { + "episode": 4464, + "epoch": 0.22545454545454546, + "eps": 0, + "loss/policy_avg": -0.4756928086280823, + "loss/value_avg": 0.47323668003082275, + "lr": 9.190440524459203e-06, + "objective/entropy": -138.91981506347656, + "objective/kl": 12.764959335327148, + "objective/non_score_reward": -0.638248085975647, + "objective/rlhf_reward": 9.533626556396484, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007105986005626619, + "policy/clipfrac_avg": 0.01297900639474392, + "policy/entropy_avg": 0.11971688270568848, + "step": 279, + "val/clipfrac_avg": 0.08647473156452179, + "val/num_eos_tokens": 596, + "val/ratio": 0.992650032043457, + "val/ratio_var": 3.488622314762324e-05 + }, + { + "episode": 4480, + "epoch": 0.22626262626262628, + "eps": 0, + "loss/policy_avg": 0.04888363927602768, + "loss/value_avg": 0.3027896583080292, + "lr": 9.1831388100321e-06, + "objective/entropy": -100.83489227294922, + "objective/kl": 16.785228729248047, + "objective/non_score_reward": -0.8392614126205444, + "objective/rlhf_reward": 8.426363945007324, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0008614714024588466, + "policy/clipfrac_avg": 0.007987331598997116, + "policy/entropy_avg": 0.13911914825439453, + "step": 280, + "val/clipfrac_avg": 0.04620702564716339, + "val/num_eos_tokens": 27, + "val/ratio": 0.9905765056610107, + "val/ratio_var": 2.7363337721908465e-05 + }, + { + "episode": 4496, + "epoch": 0.22707070707070706, + "eps": 0, + "loss/policy_avg": -0.3152393400669098, + "loss/value_avg": 0.8453013896942139, + "lr": 9.175807242664194e-06, + "objective/entropy": -116.21156311035156, + "objective/kl": 17.810836791992188, + "objective/non_score_reward": -0.8905417919158936, + "objective/rlhf_reward": 10.671957969665527, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.007155691273510456, + "policy/entropy_avg": 0.1354846954345703, + "step": 281, + "val/clipfrac_avg": 0.17372316122055054, + "val/num_eos_tokens": 248, + "val/ratio": 0.9912713766098022, + "val/ratio_var": 4.6165401727193967e-05 + }, + { + "episode": 4512, + "epoch": 0.22787878787878788, + "eps": 0, + "loss/policy_avg": 0.2918892800807953, + "loss/value_avg": 1.0024628639221191, + "lr": 9.168445874677168e-06, + "objective/entropy": -123.32449340820312, + "objective/kl": 15.429478645324707, + "objective/non_score_reward": -0.7714740037918091, + "objective/rlhf_reward": 9.443368911743164, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0007769160438328981, + "policy/clipfrac_avg": 0.01072020549327135, + "policy/entropy_avg": 0.12940633296966553, + "step": 282, + "val/clipfrac_avg": 0.1120821088552475, + "val/num_eos_tokens": 43, + "val/ratio": 0.9912656545639038, + "val/ratio_var": 3.813920193351805e-05 + }, + { + "episode": 4528, + "epoch": 0.2286868686868687, + "eps": 0, + "loss/policy_avg": -0.23349274694919586, + "loss/value_avg": 0.648268461227417, + "lr": 9.16105475860537e-06, + "objective/entropy": -79.49951171875, + "objective/kl": 17.246797561645508, + "objective/non_score_reward": -0.8623398542404175, + "objective/rlhf_reward": 7.868128776550293, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.0008569798083044589, + "policy/clipfrac_avg": 0.028804132714867592, + "policy/entropy_avg": 0.13358783721923828, + "step": 283, + "val/clipfrac_avg": 0.1429285705089569, + "val/num_eos_tokens": 444, + "val/ratio": 0.9911713004112244, + "val/ratio_var": 4.6175486204447225e-05 + }, + { + "episode": 4544, + "epoch": 0.2294949494949495, + "eps": 0, + "loss/policy_avg": 0.12218321859836578, + "loss/value_avg": 0.19281774759292603, + "lr": 9.15363394719545e-06, + "objective/entropy": -95.04164123535156, + "objective/kl": 22.310144424438477, + "objective/non_score_reward": -1.1155072450637817, + "objective/rlhf_reward": 8.868867874145508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0010480072814971209, + "policy/clipfrac_avg": 0.010950624942779541, + "policy/entropy_avg": 0.16732120513916016, + "step": 284, + "val/clipfrac_avg": 0.13048362731933594, + "val/num_eos_tokens": 203, + "val/ratio": 0.9890958070755005, + "val/ratio_var": 4.823869312531315e-05 + }, + { + "episode": 4560, + "epoch": 0.23030303030303031, + "eps": 0, + "loss/policy_avg": -0.002362152561545372, + "loss/value_avg": 0.5866045355796814, + "lr": 9.146183493405976e-06, + "objective/entropy": -42.42488098144531, + "objective/kl": 20.045467376708984, + "objective/non_score_reward": -1.0022735595703125, + "objective/rlhf_reward": 7.6539764404296875, + "objective/scores": 8.625, + "policy/approxkl_avg": 0.0008848682045936584, + "policy/clipfrac_avg": 0.007402642630040646, + "policy/entropy_avg": 0.1446681022644043, + "step": 285, + "val/clipfrac_avg": 0.1676260083913803, + "val/num_eos_tokens": 66, + "val/ratio": 0.9896621108055115, + "val/ratio_var": 2.382449383731e-05 + }, + { + "episode": 4576, + "epoch": 0.2311111111111111, + "eps": 0, + "loss/policy_avg": -0.5626534819602966, + "loss/value_avg": 1.4739155769348145, + "lr": 9.13870345040706e-06, + "objective/entropy": -106.9161376953125, + "objective/kl": 16.7766056060791, + "objective/non_score_reward": -0.8388303518295288, + "objective/rlhf_reward": 8.204137802124023, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.0008132800576277077, + "policy/clipfrac_avg": 0.0094719547778368, + "policy/entropy_avg": 0.14436912536621094, + "step": 286, + "val/clipfrac_avg": 0.1908513754606247, + "val/num_eos_tokens": 222, + "val/ratio": 0.9897047281265259, + "val/ratio_var": 4.6007939090486616e-05 + }, + { + "episode": 4592, + "epoch": 0.2319191919191919, + "eps": 0, + "loss/policy_avg": 0.1014481782913208, + "loss/value_avg": 3.819610595703125, + "lr": 9.131193871579975e-06, + "objective/entropy": -144.25914001464844, + "objective/kl": 13.762688636779785, + "objective/non_score_reward": -0.6881344318389893, + "objective/rlhf_reward": 9.99155330657959, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.002054408425465226, + "policy/clipfrac_avg": 0.009027201682329178, + "policy/entropy_avg": 0.13915538787841797, + "step": 287, + "val/clipfrac_avg": 0.21551817655563354, + "val/num_eos_tokens": 491, + "val/ratio": 0.9908062219619751, + "val/ratio_var": 0.00011211037781322375 + }, + { + "episode": 4608, + "epoch": 0.23272727272727273, + "eps": 0, + "loss/policy_avg": -0.14434215426445007, + "loss/value_avg": 1.0301947593688965, + "lr": 9.12365481051678e-06, + "objective/entropy": -86.54766082763672, + "objective/kl": 18.274917602539062, + "objective/non_score_reward": -0.9137458801269531, + "objective/rlhf_reward": 8.640941619873047, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0010243923170492053, + "policy/clipfrac_avg": 0.012742959894239902, + "policy/entropy_avg": 0.17115402221679688, + "step": 288, + "val/clipfrac_avg": 0.09064820408821106, + "val/num_eos_tokens": 80, + "val/ratio": 0.9888273477554321, + "val/ratio_var": 6.463569297920913e-05 + }, + { + "episode": 4624, + "epoch": 0.23353535353535354, + "eps": 0, + "loss/policy_avg": -0.22777478396892548, + "loss/value_avg": 0.4974277913570404, + "lr": 9.116086321019928e-06, + "objective/entropy": -80.6339111328125, + "objective/kl": 19.723041534423828, + "objective/non_score_reward": -0.9861521124839783, + "objective/rlhf_reward": 9.892754554748535, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0009434971725568175, + "policy/clipfrac_avg": 0.009782778099179268, + "policy/entropy_avg": 0.14695453643798828, + "step": 289, + "val/clipfrac_avg": 0.11158281564712524, + "val/num_eos_tokens": 422, + "val/ratio": 0.989630401134491, + "val/ratio_var": 4.67507925350219e-05 + }, + { + "episode": 4640, + "epoch": 0.23434343434343435, + "eps": 0, + "loss/policy_avg": -0.4249142110347748, + "loss/value_avg": 1.535785436630249, + "lr": 9.108488457101894e-06, + "objective/entropy": -90.72955322265625, + "objective/kl": 13.74578857421875, + "objective/non_score_reward": -0.6872894763946533, + "objective/rlhf_reward": 9.58614730834961, + "objective/scores": 10.25, + "policy/approxkl_avg": 0.0007829072419553995, + "policy/clipfrac_avg": 0.006262246984988451, + "policy/entropy_avg": 0.1394805908203125, + "step": 290, + "val/clipfrac_avg": 0.20132911205291748, + "val/num_eos_tokens": 98, + "val/ratio": 0.9905713200569153, + "val/ratio_var": 4.520507718552835e-05 + }, + { + "episode": 4656, + "epoch": 0.23515151515151514, + "eps": 0, + "loss/policy_avg": -0.009618103504180908, + "loss/value_avg": 1.8275914192199707, + "lr": 9.10086127298478e-06, + "objective/entropy": -127.0085220336914, + "objective/kl": 18.2149600982666, + "objective/non_score_reward": -0.9107480049133301, + "objective/rlhf_reward": 9.729876518249512, + "objective/scores": 10.625, + "policy/approxkl_avg": 0.0009391518542543054, + "policy/clipfrac_avg": 0.01690862700343132, + "policy/entropy_avg": 0.14864182472229004, + "step": 291, + "val/clipfrac_avg": 0.1852705329656601, + "val/num_eos_tokens": 312, + "val/ratio": 0.9907850027084351, + "val/ratio_var": 4.850202094530687e-05 + }, + { + "episode": 4672, + "epoch": 0.23595959595959595, + "eps": 0, + "loss/policy_avg": -0.3173828125, + "loss/value_avg": 0.705482542514801, + "lr": 9.093204823099932e-06, + "objective/entropy": -121.64453887939453, + "objective/kl": 17.068296432495117, + "objective/non_score_reward": -0.8534148335456848, + "objective/rlhf_reward": 9.349710464477539, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010989224538207054, + "policy/clipfrac_avg": 0.015641771256923676, + "policy/entropy_avg": 0.13405227661132812, + "step": 292, + "val/clipfrac_avg": 0.12627626955509186, + "val/num_eos_tokens": 655, + "val/ratio": 0.9925355911254883, + "val/ratio_var": 4.042615182697773e-05 + }, + { + "episode": 4688, + "epoch": 0.23676767676767677, + "eps": 0, + "loss/policy_avg": 0.2536461353302002, + "loss/value_avg": 0.8448231220245361, + "lr": 9.08551916208755e-06, + "objective/entropy": -96.61104583740234, + "objective/kl": 15.00445556640625, + "objective/non_score_reward": -0.7502228021621704, + "objective/rlhf_reward": 8.163840293884277, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0008020526147447526, + "policy/clipfrac_avg": 0.003557023126631975, + "policy/entropy_avg": 0.14197516441345215, + "step": 293, + "val/clipfrac_avg": 0.2270023375749588, + "val/num_eos_tokens": 365, + "val/ratio": 0.9892786741256714, + "val/ratio_var": 0.00012586927914526314 + }, + { + "episode": 4704, + "epoch": 0.23757575757575758, + "eps": 0, + "loss/policy_avg": -0.4800892174243927, + "loss/value_avg": 0.6021974086761475, + "lr": 9.077804344796302e-06, + "objective/entropy": -112.97701263427734, + "objective/kl": 16.631576538085938, + "objective/non_score_reward": -0.8315788507461548, + "objective/rlhf_reward": 12.012170791625977, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0006937251891940832, + "policy/clipfrac_avg": 0.004472474567592144, + "policy/entropy_avg": 0.1081966757774353, + "step": 294, + "val/clipfrac_avg": 0.32528120279312134, + "val/num_eos_tokens": 138, + "val/ratio": 0.9927380084991455, + "val/ratio_var": 4.102911407244392e-05 + }, + { + "episode": 4720, + "epoch": 0.2383838383838384, + "eps": 0, + "loss/policy_avg": 0.05609271675348282, + "loss/value_avg": 0.5157899856567383, + "lr": 9.070060426282924e-06, + "objective/entropy": -9.154275894165039, + "objective/kl": 15.853599548339844, + "objective/non_score_reward": -0.7926799654960632, + "objective/rlhf_reward": 9.293257713317871, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008039749227464199, + "policy/clipfrac_avg": 0.006328057963401079, + "policy/entropy_avg": 0.12773895263671875, + "step": 295, + "val/clipfrac_avg": 0.19731220602989197, + "val/num_eos_tokens": 17, + "val/ratio": 0.9908485412597656, + "val/ratio_var": 4.184704812359996e-05 + }, + { + "episode": 4736, + "epoch": 0.2391919191919192, + "eps": 0, + "loss/policy_avg": -0.5660660862922668, + "loss/value_avg": 1.659135341644287, + "lr": 9.06228746181184e-06, + "objective/entropy": -76.303466796875, + "objective/kl": 13.817693710327148, + "objective/non_score_reward": -0.6908847093582153, + "objective/rlhf_reward": 10.176302909851074, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008380039362236857, + "policy/clipfrac_avg": 0.018536891788244247, + "policy/entropy_avg": 0.12799948453903198, + "step": 296, + "val/clipfrac_avg": 0.31544220447540283, + "val/num_eos_tokens": 343, + "val/ratio": 0.9921280145645142, + "val/ratio_var": 4.131532114115544e-05 + }, + { + "episode": 4752, + "epoch": 0.24, + "eps": 0, + "loss/policy_avg": -0.13689611852169037, + "loss/value_avg": 0.2256709337234497, + "lr": 9.054485506854756e-06, + "objective/entropy": -127.31668853759766, + "objective/kl": 22.041820526123047, + "objective/non_score_reward": -1.102090835571289, + "objective/rlhf_reward": 7.948689937591553, + "objective/scores": 9.0625, + "policy/approxkl_avg": 0.000870460004080087, + "policy/clipfrac_avg": 0.008635717444121838, + "policy/entropy_avg": 0.14861583709716797, + "step": 297, + "val/clipfrac_avg": 0.06009959429502487, + "val/num_eos_tokens": 572, + "val/ratio": 0.9896406531333923, + "val/ratio_var": 3.543778802850284e-05 + }, + { + "episode": 4768, + "epoch": 0.2408080808080808, + "eps": 0, + "loss/policy_avg": -0.287416934967041, + "loss/value_avg": 1.405874490737915, + "lr": 9.046654617090267e-06, + "objective/entropy": -99.76239013671875, + "objective/kl": 17.812686920166016, + "objective/non_score_reward": -0.8906343579292297, + "objective/rlhf_reward": 10.421865463256836, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0007631564512848854, + "policy/clipfrac_avg": 0.010343081317842007, + "policy/entropy_avg": 0.12606573104858398, + "step": 298, + "val/clipfrac_avg": 0.16891875863075256, + "val/num_eos_tokens": 157, + "val/ratio": 0.9915649890899658, + "val/ratio_var": 3.898761860909872e-05 + }, + { + "episode": 4784, + "epoch": 0.24161616161616162, + "eps": 0, + "loss/policy_avg": -0.3347007930278778, + "loss/value_avg": 1.4577244520187378, + "lr": 9.038794848403463e-06, + "objective/entropy": -53.73211669921875, + "objective/kl": 18.920724868774414, + "objective/non_score_reward": -0.9460362195968628, + "objective/rlhf_reward": 8.979745864868164, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0008735101437196136, + "policy/clipfrac_avg": 0.005150888580828905, + "policy/entropy_avg": 0.15177440643310547, + "step": 299, + "val/clipfrac_avg": 0.07536730915307999, + "val/num_eos_tokens": 184, + "val/ratio": 0.9899022579193115, + "val/ratio_var": 3.8399972254410386e-05 + }, + { + "episode": 4800, + "epoch": 0.24242424242424243, + "eps": 0, + "loss/policy_avg": -0.16605234146118164, + "loss/value_avg": 0.31406185030937195, + "lr": 9.030906256885528e-06, + "objective/entropy": -111.41056823730469, + "objective/kl": 18.138174057006836, + "objective/non_score_reward": -0.9069086313247681, + "objective/rlhf_reward": 6.542309761047363, + "objective/scores": 7.4375, + "policy/approxkl_avg": 0.000873133132699877, + "policy/clipfrac_avg": 0.021350745111703873, + "policy/entropy_avg": 0.12722575664520264, + "step": 300, + "val/clipfrac_avg": 0.05479028448462486, + "val/num_eos_tokens": 420, + "val/ratio": 0.9915505647659302, + "val/ratio_var": 4.506599725573324e-05 + }, + { + "episode": 4816, + "epoch": 0.24323232323232324, + "eps": 0, + "loss/policy_avg": -0.0382872149348259, + "loss/value_avg": 0.3191486597061157, + "lr": 9.022988898833342e-06, + "objective/entropy": -129.36647033691406, + "objective/kl": 20.153329849243164, + "objective/non_score_reward": -1.0076665878295898, + "objective/rlhf_reward": 9.46889591217041, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008897838997654617, + "policy/clipfrac_avg": 0.01343005895614624, + "policy/entropy_avg": 0.13002967834472656, + "step": 301, + "val/clipfrac_avg": 0.08865603804588318, + "val/num_eos_tokens": 169, + "val/ratio": 0.990653395652771, + "val/ratio_var": 4.155634087510407e-05 + }, + { + "episode": 4832, + "epoch": 0.24404040404040403, + "eps": 0, + "loss/policy_avg": -0.013936442323029041, + "loss/value_avg": 0.2754359245300293, + "lr": 9.015042830749075e-06, + "objective/entropy": -88.79689025878906, + "objective/kl": 23.026803970336914, + "objective/non_score_reward": -1.1513402462005615, + "objective/rlhf_reward": 8.06740951538086, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0007821049075573683, + "policy/clipfrac_avg": 0.006361945066601038, + "policy/entropy_avg": 0.147918701171875, + "step": 302, + "val/clipfrac_avg": 0.08944191783666611, + "val/num_eos_tokens": 586, + "val/ratio": 0.9910757541656494, + "val/ratio_var": 2.4175893486244604e-05 + }, + { + "episode": 4848, + "epoch": 0.24484848484848484, + "eps": 0, + "loss/policy_avg": 0.16253715753555298, + "loss/value_avg": 0.3962807059288025, + "lr": 9.007068109339783e-06, + "objective/entropy": -153.9242401123047, + "objective/kl": 24.887285232543945, + "objective/non_score_reward": -1.2443642616271973, + "objective/rlhf_reward": 7.620870113372803, + "objective/scores": 8.875, + "policy/approxkl_avg": 0.0010797552531585097, + "policy/clipfrac_avg": 0.020701207220554352, + "policy/entropy_avg": 0.14539474248886108, + "step": 303, + "val/clipfrac_avg": 0.09651350975036621, + "val/num_eos_tokens": 141, + "val/ratio": 0.9895668029785156, + "val/ratio_var": 5.088368197903037e-05 + }, + { + "episode": 4864, + "epoch": 0.24565656565656566, + "eps": 0, + "loss/policy_avg": 0.3374157249927521, + "loss/value_avg": 3.7259681224823, + "lr": 8.999064791517013e-06, + "objective/entropy": -112.62982177734375, + "objective/kl": 18.144006729125977, + "objective/non_score_reward": -0.9072003364562988, + "objective/rlhf_reward": 9.682642936706543, + "objective/scores": 10.5625, + "policy/approxkl_avg": 0.0008159844437614083, + "policy/clipfrac_avg": 0.01846667192876339, + "policy/entropy_avg": 0.1183176040649414, + "step": 304, + "val/clipfrac_avg": 0.17207640409469604, + "val/num_eos_tokens": 604, + "val/ratio": 0.9920910596847534, + "val/ratio_var": 2.8615042538149282e-05 + }, + { + "episode": 4880, + "epoch": 0.24646464646464647, + "eps": 0, + "loss/policy_avg": 0.14597804844379425, + "loss/value_avg": 0.4686759412288666, + "lr": 8.991032934396386e-06, + "objective/entropy": -110.00303649902344, + "objective/kl": 21.02062225341797, + "objective/non_score_reward": -1.0510311126708984, + "objective/rlhf_reward": 9.300531387329102, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009549619280733168, + "policy/clipfrac_avg": 0.022998899221420288, + "policy/entropy_avg": 0.14881080389022827, + "step": 305, + "val/clipfrac_avg": 0.1400592029094696, + "val/num_eos_tokens": 730, + "val/ratio": 0.9895407557487488, + "val/ratio_var": 4.939505743095651e-05 + }, + { + "episode": 4896, + "epoch": 0.24727272727272728, + "eps": 0, + "loss/policy_avg": -0.09728778898715973, + "loss/value_avg": 0.6937114000320435, + "lr": 8.982972595297195e-06, + "objective/entropy": -137.5471649169922, + "objective/kl": 19.597259521484375, + "objective/non_score_reward": -0.9798629283905029, + "objective/rlhf_reward": 8.938105583190918, + "objective/scores": 9.9375, + "policy/approxkl_avg": 0.0007075261091813445, + "policy/clipfrac_avg": 0.013334396295249462, + "policy/entropy_avg": 0.12405538558959961, + "step": 306, + "val/clipfrac_avg": 0.2546675205230713, + "val/num_eos_tokens": 467, + "val/ratio": 0.9915751218795776, + "val/ratio_var": 4.6088196540949866e-05 + }, + { + "episode": 4912, + "epoch": 0.24808080808080807, + "eps": 0, + "loss/policy_avg": 0.1869451254606247, + "loss/value_avg": 0.5242486000061035, + "lr": 8.97488383174199e-06, + "objective/entropy": -90.83536529541016, + "objective/kl": 16.863101959228516, + "objective/non_score_reward": -0.8431550860404968, + "objective/rlhf_reward": 10.844345092773438, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0007314659305848181, + "policy/clipfrac_avg": 0.007252814248204231, + "policy/entropy_avg": 0.15680980682373047, + "step": 307, + "val/clipfrac_avg": 0.2522747218608856, + "val/num_eos_tokens": 632, + "val/ratio": 0.9911881685256958, + "val/ratio_var": 4.907942275167443e-05 + }, + { + "episode": 4928, + "epoch": 0.24888888888888888, + "eps": 0, + "loss/policy_avg": -0.05933833122253418, + "loss/value_avg": 0.7149787545204163, + "lr": 8.966766701456177e-06, + "objective/entropy": -96.56021118164062, + "objective/kl": 19.828670501708984, + "objective/non_score_reward": -0.9914336204528809, + "objective/rlhf_reward": 8.750753402709961, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.001207109191454947, + "policy/clipfrac_avg": 0.027725519612431526, + "policy/entropy_avg": 0.14550870656967163, + "step": 308, + "val/clipfrac_avg": 0.07701820880174637, + "val/num_eos_tokens": 384, + "val/ratio": 0.9902600049972534, + "val/ratio_var": 5.600972872343846e-05 + }, + { + "episode": 4944, + "epoch": 0.2496969696969697, + "eps": 0, + "loss/policy_avg": -0.45973384380340576, + "loss/value_avg": 0.9564704895019531, + "lr": 8.9586212623676e-06, + "objective/entropy": -98.84840393066406, + "objective/kl": 21.63240623474121, + "objective/non_score_reward": -1.081620454788208, + "objective/rlhf_reward": 11.062911033630371, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.000988299841992557, + "policy/clipfrac_avg": 0.021323896944522858, + "policy/entropy_avg": 0.15676051378250122, + "step": 309, + "val/clipfrac_avg": 0.1754695475101471, + "val/num_eos_tokens": 12, + "val/ratio": 0.9899306893348694, + "val/ratio_var": 8.19729539216496e-05 + }, + { + "episode": 4960, + "epoch": 0.2505050505050505, + "eps": 0, + "loss/policy_avg": -0.03560265153646469, + "loss/value_avg": 1.048866629600525, + "lr": 8.950447572606128e-06, + "objective/entropy": -54.279483795166016, + "objective/kl": 19.034969329833984, + "objective/non_score_reward": -0.9517484903335571, + "objective/rlhf_reward": 8.87637710571289, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0012411081697791815, + "policy/clipfrac_avg": 0.005438921041786671, + "policy/entropy_avg": 0.1564793586730957, + "step": 310, + "val/clipfrac_avg": 0.19135892391204834, + "val/num_eos_tokens": 3, + "val/ratio": 0.9885050058364868, + "val/ratio_var": 7.225138688227162e-05 + }, + { + "episode": 4976, + "epoch": 0.2513131313131313, + "eps": 0, + "loss/policy_avg": 0.022867269814014435, + "loss/value_avg": 0.9407914876937866, + "lr": 8.94224569050324e-06, + "objective/entropy": -90.46046447753906, + "objective/kl": 16.922504425048828, + "objective/non_score_reward": -0.8461253643035889, + "objective/rlhf_reward": 8.325750350952148, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0010159123921766877, + "policy/clipfrac_avg": 0.01187141053378582, + "policy/entropy_avg": 0.15433311462402344, + "step": 311, + "val/clipfrac_avg": 0.22986651957035065, + "val/num_eos_tokens": 578, + "val/ratio": 0.9906753301620483, + "val/ratio_var": 6.91376335453242e-05 + }, + { + "episode": 4992, + "epoch": 0.25212121212121213, + "eps": 0, + "loss/policy_avg": -0.29373523592948914, + "loss/value_avg": 0.7329980731010437, + "lr": 8.93401567459161e-06, + "objective/entropy": -124.9230728149414, + "objective/kl": 18.02437400817871, + "objective/non_score_reward": -0.9012187719345093, + "objective/rlhf_reward": 9.07143783569336, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.0007548386347480118, + "policy/clipfrac_avg": 0.01587068662047386, + "policy/entropy_avg": 0.12101477384567261, + "step": 312, + "val/clipfrac_avg": 0.1042708307504654, + "val/num_eos_tokens": 173, + "val/ratio": 0.9913851022720337, + "val/ratio_var": 5.3006267989985645e-05 + }, + { + "episode": 5008, + "epoch": 0.25292929292929295, + "eps": 0, + "loss/policy_avg": 0.26103121042251587, + "loss/value_avg": 0.7720507383346558, + "lr": 8.92575758360469e-06, + "objective/entropy": -52.38456344604492, + "objective/kl": 17.74087142944336, + "objective/non_score_reward": -0.8870435953140259, + "objective/rlhf_reward": 9.171550750732422, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0007087467820383608, + "policy/clipfrac_avg": 0.008573219180107117, + "policy/entropy_avg": 0.12889361381530762, + "step": 313, + "val/clipfrac_avg": 0.165409117937088, + "val/num_eos_tokens": 54, + "val/ratio": 0.9908913373947144, + "val/ratio_var": 1.9720375348697416e-05 + }, + { + "episode": 5024, + "epoch": 0.25373737373737376, + "eps": 0, + "loss/policy_avg": -0.0015758797526359558, + "loss/value_avg": 0.2577587962150574, + "lr": 8.91747147647629e-06, + "objective/entropy": -135.29092407226562, + "objective/kl": 20.900075912475586, + "objective/non_score_reward": -1.0450037717819214, + "objective/rlhf_reward": 9.150308609008789, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0006939490558579564, + "policy/clipfrac_avg": 0.0060662562027573586, + "policy/entropy_avg": 0.12897872924804688, + "step": 314, + "val/clipfrac_avg": 0.1105986088514328, + "val/num_eos_tokens": 75, + "val/ratio": 0.9913695454597473, + "val/ratio_var": 4.320408697822131e-05 + }, + { + "episode": 5040, + "epoch": 0.2545454545454545, + "eps": 0, + "loss/policy_avg": 0.1372879296541214, + "loss/value_avg": 1.6457445621490479, + "lr": 8.90915741234015e-06, + "objective/entropy": -87.80855560302734, + "objective/kl": 17.859722137451172, + "objective/non_score_reward": -0.8929861187934875, + "objective/rlhf_reward": 10.048419952392578, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0010231503983959556, + "policy/clipfrac_avg": 0.005694298073649406, + "policy/entropy_avg": 0.18284660577774048, + "step": 315, + "val/clipfrac_avg": 0.20232242345809937, + "val/num_eos_tokens": 37, + "val/ratio": 0.9879668354988098, + "val/ratio_var": 5.564819002756849e-05 + }, + { + "episode": 5056, + "epoch": 0.25535353535353533, + "eps": 0, + "loss/policy_avg": -0.03426007553935051, + "loss/value_avg": 0.335923969745636, + "lr": 8.900815450529536e-06, + "objective/entropy": -81.0003890991211, + "objective/kl": 18.628372192382812, + "objective/non_score_reward": -0.9314186573028564, + "objective/rlhf_reward": 8.279519081115723, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008206713246181607, + "policy/clipfrac_avg": 0.005000715609639883, + "policy/entropy_avg": 0.14201807975769043, + "step": 316, + "val/clipfrac_avg": 0.08686406910419464, + "val/num_eos_tokens": 117, + "val/ratio": 0.9906059503555298, + "val/ratio_var": 4.339249062468298e-05 + }, + { + "episode": 5072, + "epoch": 0.25616161616161615, + "eps": 0, + "loss/policy_avg": -0.08662296831607819, + "loss/value_avg": 0.6432848572731018, + "lr": 8.892445650576796e-06, + "objective/entropy": -72.72438049316406, + "objective/kl": 18.402015686035156, + "objective/non_score_reward": -0.9201008081436157, + "objective/rlhf_reward": 9.064273834228516, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.002800849499180913, + "policy/clipfrac_avg": 0.018393252044916153, + "policy/entropy_avg": 0.14298725128173828, + "step": 317, + "val/clipfrac_avg": 0.24806933104991913, + "val/num_eos_tokens": 137, + "val/ratio": 0.9901000261306763, + "val/ratio_var": 6.166585080791265e-05 + }, + { + "episode": 5088, + "epoch": 0.25696969696969696, + "eps": 0, + "loss/policy_avg": -0.31295180320739746, + "loss/value_avg": 0.8796172142028809, + "lr": 8.884048072212952e-06, + "objective/entropy": -108.95730590820312, + "objective/kl": 18.707744598388672, + "objective/non_score_reward": -0.9353872537612915, + "objective/rlhf_reward": 8.502113342285156, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0008305341470986605, + "policy/clipfrac_avg": 0.028556479141116142, + "policy/entropy_avg": 0.13506406545639038, + "step": 318, + "val/clipfrac_avg": 0.08825132250785828, + "val/num_eos_tokens": 314, + "val/ratio": 0.9902423620223999, + "val/ratio_var": 4.728272688225843e-05 + }, + { + "episode": 5104, + "epoch": 0.2577777777777778, + "eps": 0, + "loss/policy_avg": 0.38284558057785034, + "loss/value_avg": 1.085472822189331, + "lr": 8.87562277536726e-06, + "objective/entropy": -173.64369201660156, + "objective/kl": 22.074050903320312, + "objective/non_score_reward": -1.1037025451660156, + "objective/rlhf_reward": 7.716609954833984, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.000958627148065716, + "policy/clipfrac_avg": 0.052636995911598206, + "policy/entropy_avg": 0.1379566192626953, + "step": 319, + "val/clipfrac_avg": 0.18396008014678955, + "val/num_eos_tokens": 46, + "val/ratio": 0.9896219372749329, + "val/ratio_var": 5.951917773927562e-05 + }, + { + "episode": 5120, + "epoch": 0.2585858585858586, + "eps": 0, + "loss/policy_avg": -0.012532621622085571, + "loss/value_avg": 2.471778154373169, + "lr": 8.867169820166794e-06, + "objective/entropy": -72.43736267089844, + "objective/kl": 15.281889915466309, + "objective/non_score_reward": -0.7640944719314575, + "objective/rlhf_reward": 11.649968147277832, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.001308539416640997, + "policy/clipfrac_avg": 0.038986120373010635, + "policy/entropy_avg": 0.1654043197631836, + "step": 320, + "val/clipfrac_avg": 0.3568417429924011, + "val/num_eos_tokens": 403, + "val/ratio": 0.9890977144241333, + "val/ratio_var": 0.0001693846279522404 + }, + { + "episode": 5136, + "epoch": 0.2593939393939394, + "eps": 0, + "loss/policy_avg": 0.09475673735141754, + "loss/value_avg": 1.4181828498840332, + "lr": 8.85868926693601e-06, + "objective/entropy": -111.93338775634766, + "objective/kl": 15.290900230407715, + "objective/non_score_reward": -0.7645449638366699, + "objective/rlhf_reward": 8.227643013000488, + "objective/scores": 9.0, + "policy/approxkl_avg": 0.0012109801173210144, + "policy/clipfrac_avg": 0.014719913713634014, + "policy/entropy_avg": 0.2194991111755371, + "step": 321, + "val/clipfrac_avg": 0.23112055659294128, + "val/num_eos_tokens": 202, + "val/ratio": 0.9894407987594604, + "val/ratio_var": 0.0001930692815221846 + }, + { + "episode": 5152, + "epoch": 0.2602020202020202, + "eps": 0, + "loss/policy_avg": -0.03570065647363663, + "loss/value_avg": 0.6953660249710083, + "lr": 8.850181176196316e-06, + "objective/entropy": -67.42729949951172, + "objective/kl": 20.265304565429688, + "objective/non_score_reward": -1.0132652521133423, + "objective/rlhf_reward": 9.896890640258789, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.001000801450572908, + "policy/clipfrac_avg": 0.013037353754043579, + "policy/entropy_avg": 0.15838980674743652, + "step": 322, + "val/clipfrac_avg": 0.2150072306394577, + "val/num_eos_tokens": 316, + "val/ratio": 0.9900329113006592, + "val/ratio_var": 4.196034205961041e-05 + }, + { + "episode": 5168, + "epoch": 0.261010101010101, + "eps": 0, + "loss/policy_avg": -0.10239146649837494, + "loss/value_avg": 0.11446619033813477, + "lr": 8.84164560866564e-06, + "objective/entropy": -34.665523529052734, + "objective/kl": 20.744216918945312, + "objective/non_score_reward": -1.0372107028961182, + "objective/rlhf_reward": 8.302632331848145, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010730468202382326, + "policy/clipfrac_avg": 0.00638403557240963, + "policy/entropy_avg": 0.172841876745224, + "step": 323, + "val/clipfrac_avg": 0.04469859600067139, + "val/num_eos_tokens": 82, + "val/ratio": 0.9883930683135986, + "val/ratio_var": 3.371910861460492e-05 + }, + { + "episode": 5184, + "epoch": 0.26181818181818184, + "eps": 0, + "loss/policy_avg": -0.12644696235656738, + "loss/value_avg": 0.28114426136016846, + "lr": 8.833082625258003e-06, + "objective/entropy": -215.53665161132812, + "objective/kl": 21.241905212402344, + "objective/non_score_reward": -1.0620954036712646, + "objective/rlhf_reward": 8.551185607910156, + "objective/scores": 9.625, + "policy/approxkl_avg": 0.000840157619677484, + "policy/clipfrac_avg": 0.006537843961268663, + "policy/entropy_avg": 0.15298840403556824, + "step": 324, + "val/clipfrac_avg": 0.04681334272027016, + "val/num_eos_tokens": 128, + "val/ratio": 0.9906848669052124, + "val/ratio_var": 3.288772495579906e-05 + }, + { + "episode": 5200, + "epoch": 0.26262626262626265, + "eps": 0, + "loss/policy_avg": 0.32401522994041443, + "loss/value_avg": 0.6005299687385559, + "lr": 8.824492287083072e-06, + "objective/entropy": -130.52597045898438, + "objective/kl": 17.67336654663086, + "objective/non_score_reward": -0.8836683034896851, + "objective/rlhf_reward": 8.811644554138184, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0012680396903306246, + "policy/clipfrac_avg": 0.007880561985075474, + "policy/entropy_avg": 0.1447734832763672, + "step": 325, + "val/clipfrac_avg": 0.19150397181510925, + "val/num_eos_tokens": 618, + "val/ratio": 0.9905928373336792, + "val/ratio_var": 5.435891216620803e-05 + }, + { + "episode": 5216, + "epoch": 0.2634343434343434, + "eps": 0, + "loss/policy_avg": -0.15733838081359863, + "loss/value_avg": 0.5087392926216125, + "lr": 8.815874655445742e-06, + "objective/entropy": -127.53799438476562, + "objective/kl": 18.464536666870117, + "objective/non_score_reward": -0.9232268333435059, + "objective/rlhf_reward": 7.912710666656494, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008032459882088006, + "policy/clipfrac_avg": 0.006318090949207544, + "policy/entropy_avg": 0.14925909042358398, + "step": 326, + "val/clipfrac_avg": 0.08119470626115799, + "val/num_eos_tokens": 386, + "val/ratio": 0.9900246858596802, + "val/ratio_var": 4.682234794017859e-05 + }, + { + "episode": 5232, + "epoch": 0.2642424242424242, + "eps": 0, + "loss/policy_avg": -0.4491274952888489, + "loss/value_avg": 0.6588542461395264, + "lr": 8.807229791845673e-06, + "objective/entropy": -80.7568359375, + "objective/kl": 20.631011962890625, + "objective/non_score_reward": -1.0315505266189575, + "objective/rlhf_reward": 8.792668342590332, + "objective/scores": 9.8125, + "policy/approxkl_avg": 0.0008535610395483673, + "policy/clipfrac_avg": 0.020397143438458443, + "policy/entropy_avg": 0.14399003982543945, + "step": 327, + "val/clipfrac_avg": 0.06285601854324341, + "val/num_eos_tokens": 202, + "val/ratio": 0.9904580116271973, + "val/ratio_var": 3.422212466830388e-05 + }, + { + "episode": 5248, + "epoch": 0.26505050505050504, + "eps": 0, + "loss/policy_avg": -0.3479881286621094, + "loss/value_avg": 1.1624462604522705, + "lr": 8.798557757976877e-06, + "objective/entropy": -80.94671630859375, + "objective/kl": 20.599206924438477, + "objective/non_score_reward": -1.0299603939056396, + "objective/rlhf_reward": 9.352851867675781, + "objective/scores": 10.375, + "policy/approxkl_avg": 0.0009967453079298139, + "policy/clipfrac_avg": 0.023027844727039337, + "policy/entropy_avg": 0.15660107135772705, + "step": 328, + "val/clipfrac_avg": 0.16796532273292542, + "val/num_eos_tokens": 565, + "val/ratio": 0.9897423982620239, + "val/ratio_var": 6.0243080952204764e-05 + }, + { + "episode": 5264, + "epoch": 0.26585858585858585, + "eps": 0, + "loss/policy_avg": -0.02400904893875122, + "loss/value_avg": 1.3643267154693604, + "lr": 8.789858615727266e-06, + "objective/entropy": -106.59388732910156, + "objective/kl": 20.202247619628906, + "objective/non_score_reward": -1.0101124048233032, + "objective/rlhf_reward": 10.064105987548828, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0008892196929082274, + "policy/clipfrac_avg": 0.01402290165424347, + "policy/entropy_avg": 0.15981292724609375, + "step": 329, + "val/clipfrac_avg": 0.11122452467679977, + "val/num_eos_tokens": 46, + "val/ratio": 0.9890484809875488, + "val/ratio_var": 6.913124525453895e-05 + }, + { + "episode": 5280, + "epoch": 0.26666666666666666, + "eps": 0, + "loss/policy_avg": -0.06329865008592606, + "loss/value_avg": 0.5471635460853577, + "lr": 8.781132427178203e-06, + "objective/entropy": -62.446468353271484, + "objective/kl": 17.81369972229004, + "objective/non_score_reward": -0.890684962272644, + "objective/rlhf_reward": 8.996033668518066, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0006900709122419357, + "policy/clipfrac_avg": 0.004856608342379332, + "policy/entropy_avg": 0.1437549591064453, + "step": 330, + "val/clipfrac_avg": 0.1539391279220581, + "val/num_eos_tokens": 7, + "val/ratio": 0.9915682077407837, + "val/ratio_var": 2.577338455012068e-05 + }, + { + "episode": 5296, + "epoch": 0.2674747474747475, + "eps": 0, + "loss/policy_avg": 0.6682453751564026, + "loss/value_avg": 2.5147206783294678, + "lr": 8.772379254604074e-06, + "objective/entropy": -88.05587768554688, + "objective/kl": 19.03775405883789, + "objective/non_score_reward": -0.9518877267837524, + "objective/rlhf_reward": 6.600846767425537, + "objective/scores": 7.5625, + "policy/approxkl_avg": 0.0008258739835582674, + "policy/clipfrac_avg": 0.045845311135053635, + "policy/entropy_avg": 0.12726306915283203, + "step": 331, + "val/clipfrac_avg": 0.24742355942726135, + "val/num_eos_tokens": 210, + "val/ratio": 0.9908438920974731, + "val/ratio_var": 3.244361505494453e-05 + }, + { + "episode": 5312, + "epoch": 0.2682828282828283, + "eps": 0, + "loss/policy_avg": -0.07922033965587616, + "loss/value_avg": 0.5227113962173462, + "lr": 8.763599160471831e-06, + "objective/entropy": -166.76419067382812, + "objective/kl": 16.866302490234375, + "objective/non_score_reward": -0.843315064907074, + "objective/rlhf_reward": 9.953559875488281, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0008331661229021847, + "policy/clipfrac_avg": 0.0146230673417449, + "policy/entropy_avg": 0.1378493309020996, + "step": 332, + "val/clipfrac_avg": 0.21186737716197968, + "val/num_eos_tokens": 293, + "val/ratio": 0.9917409420013428, + "val/ratio_var": 4.541428279480897e-05 + }, + { + "episode": 5328, + "epoch": 0.2690909090909091, + "eps": 0, + "loss/policy_avg": -0.10379679501056671, + "loss/value_avg": 0.22473345696926117, + "lr": 8.754792207440557e-06, + "objective/entropy": -113.41132354736328, + "objective/kl": 23.771095275878906, + "objective/non_score_reward": -1.1885547637939453, + "objective/rlhf_reward": 8.971601486206055, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010298178531229496, + "policy/clipfrac_avg": 0.008960915729403496, + "policy/entropy_avg": 0.15825366973876953, + "step": 333, + "val/clipfrac_avg": 0.07536853849887848, + "val/num_eos_tokens": 622, + "val/ratio": 0.9886398315429688, + "val/ratio_var": 8.407732821069658e-05 + }, + { + "episode": 5344, + "epoch": 0.2698989898989899, + "eps": 0, + "loss/policy_avg": 0.2412254959344864, + "loss/value_avg": 0.5594663619995117, + "lr": 8.745958458361011e-06, + "objective/entropy": -110.30827331542969, + "objective/kl": 12.800091743469238, + "objective/non_score_reward": -0.640004575252533, + "objective/rlhf_reward": 8.453744888305664, + "objective/scores": 9.125, + "policy/approxkl_avg": 0.0006719123339280486, + "policy/clipfrac_avg": 0.009869431145489216, + "policy/entropy_avg": 0.10278797149658203, + "step": 334, + "val/clipfrac_avg": 0.2574247419834137, + "val/num_eos_tokens": 260, + "val/ratio": 0.9938614368438721, + "val/ratio_var": 4.6416309487540275e-05 + }, + { + "episode": 5360, + "epoch": 0.27070707070707073, + "eps": 0, + "loss/policy_avg": -0.02042115107178688, + "loss/value_avg": 0.5813127756118774, + "lr": 8.737097976275177e-06, + "objective/entropy": -132.7894287109375, + "objective/kl": 20.533952713012695, + "objective/non_score_reward": -1.0266976356506348, + "objective/rlhf_reward": 8.508459091186523, + "objective/scores": 9.5625, + "policy/approxkl_avg": 0.0007548570865765214, + "policy/clipfrac_avg": 0.007180739659816027, + "policy/entropy_avg": 0.14557290077209473, + "step": 335, + "val/clipfrac_avg": 0.12942752242088318, + "val/num_eos_tokens": 910, + "val/ratio": 0.9915430545806885, + "val/ratio_var": 3.247900531277992e-05 + }, + { + "episode": 5376, + "epoch": 0.27151515151515154, + "eps": 0, + "loss/policy_avg": -0.2686082422733307, + "loss/value_avg": 1.6656913757324219, + "lr": 8.728210824415829e-06, + "objective/entropy": -78.77982330322266, + "objective/kl": 20.93308448791504, + "objective/non_score_reward": -1.046654224395752, + "objective/rlhf_reward": 9.949440002441406, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010565763805061579, + "policy/clipfrac_avg": 0.012101948261260986, + "policy/entropy_avg": 0.15659451484680176, + "step": 336, + "val/clipfrac_avg": 0.12851554155349731, + "val/num_eos_tokens": 198, + "val/ratio": 0.9906046390533447, + "val/ratio_var": 3.88691078114789e-05 + }, + { + "episode": 5392, + "epoch": 0.2723232323232323, + "eps": 0, + "loss/policy_avg": 0.24045519530773163, + "loss/value_avg": 1.6145107746124268, + "lr": 8.71929706620606e-06, + "objective/entropy": -131.0155487060547, + "objective/kl": 16.704288482666016, + "objective/non_score_reward": -0.835214376449585, + "objective/rlhf_reward": 10.223379135131836, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0006117980228736997, + "policy/clipfrac_avg": 0.0048257140442729, + "policy/entropy_avg": 0.10847282409667969, + "step": 337, + "val/clipfrac_avg": 0.15292316675186157, + "val/num_eos_tokens": 780, + "val/ratio": 0.9926360845565796, + "val/ratio_var": 4.497100235312246e-05 + }, + { + "episode": 5408, + "epoch": 0.2731313131313131, + "eps": 0, + "loss/policy_avg": 0.18228551745414734, + "loss/value_avg": 2.502713680267334, + "lr": 8.710356765258845e-06, + "objective/entropy": -91.27498626708984, + "objective/kl": 18.50728988647461, + "objective/non_score_reward": -0.9253645539283752, + "objective/rlhf_reward": 10.207448959350586, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000849106814712286, + "policy/clipfrac_avg": 0.007296066731214523, + "policy/entropy_avg": 0.14929378032684326, + "step": 338, + "val/clipfrac_avg": 0.16338565945625305, + "val/num_eos_tokens": 195, + "val/ratio": 0.9899997711181641, + "val/ratio_var": 5.115730164106935e-05 + }, + { + "episode": 5424, + "epoch": 0.2739393939393939, + "eps": 0, + "loss/policy_avg": 0.10175848752260208, + "loss/value_avg": 1.1190319061279297, + "lr": 8.701389985376578e-06, + "objective/entropy": -85.34510803222656, + "objective/kl": 20.304767608642578, + "objective/non_score_reward": -1.0152382850646973, + "objective/rlhf_reward": 10.492574691772461, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.000908980960957706, + "policy/clipfrac_avg": 0.011570381000638008, + "policy/entropy_avg": 0.16573619842529297, + "step": 339, + "val/clipfrac_avg": 0.1429380476474762, + "val/num_eos_tokens": 772, + "val/ratio": 0.9889801144599915, + "val/ratio_var": 5.0471229769755155e-05 + }, + { + "episode": 5440, + "epoch": 0.27474747474747474, + "eps": 0, + "loss/policy_avg": 0.14838971197605133, + "loss/value_avg": 4.23499870300293, + "lr": 8.692396790550621e-06, + "objective/entropy": -81.48554229736328, + "objective/kl": 19.259021759033203, + "objective/non_score_reward": -0.9629510641098022, + "objective/rlhf_reward": 9.72454833984375, + "objective/scores": 10.6875, + "policy/approxkl_avg": 0.0008599548600614071, + "policy/clipfrac_avg": 0.013572474010288715, + "policy/entropy_avg": 0.12827253341674805, + "step": 340, + "val/clipfrac_avg": 0.09922881424427032, + "val/num_eos_tokens": 42, + "val/ratio": 0.9914301633834839, + "val/ratio_var": 4.035350139020011e-05 + }, + { + "episode": 5456, + "epoch": 0.27555555555555555, + "eps": 0, + "loss/policy_avg": -0.4641180634498596, + "loss/value_avg": 0.893822193145752, + "lr": 8.683377244960847e-06, + "objective/entropy": -45.80091857910156, + "objective/kl": 23.56014633178711, + "objective/non_score_reward": -1.1780073642730713, + "objective/rlhf_reward": 9.849336624145508, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010738306446000934, + "policy/clipfrac_avg": 0.00759343383833766, + "policy/entropy_avg": 0.19598770141601562, + "step": 341, + "val/clipfrac_avg": 0.23792031407356262, + "val/num_eos_tokens": 90, + "val/ratio": 0.9857372641563416, + "val/ratio_var": 0.00010740475408965722 + }, + { + "episode": 5472, + "epoch": 0.27636363636363637, + "eps": 0, + "loss/policy_avg": -0.7033554911613464, + "loss/value_avg": 1.0022238492965698, + "lr": 8.674331412975178e-06, + "objective/entropy": -99.86138153076172, + "objective/kl": 18.864389419555664, + "objective/non_score_reward": -0.9432195425033569, + "objective/rlhf_reward": 9.588029861450195, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.000636940763797611, + "policy/clipfrac_avg": 0.004983609076589346, + "policy/entropy_avg": 0.11473152041435242, + "step": 342, + "val/clipfrac_avg": 0.086562380194664, + "val/num_eos_tokens": 62, + "val/ratio": 0.9927915334701538, + "val/ratio_var": 2.162680175388232e-05 + }, + { + "episode": 5488, + "epoch": 0.2771717171717172, + "eps": 0, + "loss/policy_avg": 0.16984277963638306, + "loss/value_avg": 1.0195503234863281, + "lr": 8.665259359149132e-06, + "objective/entropy": -147.96414184570312, + "objective/kl": 15.84041690826416, + "objective/non_score_reward": -0.7920209169387817, + "objective/rlhf_reward": 11.223604202270508, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.001866342849098146, + "policy/clipfrac_avg": 0.015933256596326828, + "policy/entropy_avg": 0.12890291213989258, + "step": 343, + "val/clipfrac_avg": 0.16014033555984497, + "val/num_eos_tokens": 729, + "val/ratio": 0.9920316934585571, + "val/ratio_var": 3.811761780525558e-05 + }, + { + "episode": 5504, + "epoch": 0.277979797979798, + "eps": 0, + "loss/policy_avg": 0.05566444247961044, + "loss/value_avg": 0.40116769075393677, + "lr": 8.656161148225357e-06, + "objective/entropy": -118.87034606933594, + "objective/kl": 16.89540672302246, + "objective/non_score_reward": -0.8447704315185547, + "objective/rlhf_reward": 10.327104568481445, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0007699039415456355, + "policy/clipfrac_avg": 0.011794596910476685, + "policy/entropy_avg": 0.14203107357025146, + "step": 344, + "val/clipfrac_avg": 0.2546975612640381, + "val/num_eos_tokens": 453, + "val/ratio": 0.9906063079833984, + "val/ratio_var": 5.0702645239653066e-05 + }, + { + "episode": 5520, + "epoch": 0.2787878787878788, + "eps": 0, + "loss/policy_avg": -0.27140140533447266, + "loss/value_avg": 1.5131020545959473, + "lr": 8.647036845133171e-06, + "objective/entropy": -76.3680419921875, + "objective/kl": 16.241445541381836, + "objective/non_score_reward": -0.8120723366737366, + "objective/rlhf_reward": 11.08636474609375, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0007258071564137936, + "policy/clipfrac_avg": 0.005320374853909016, + "policy/entropy_avg": 0.12322688102722168, + "step": 345, + "val/clipfrac_avg": 0.14076992869377136, + "val/num_eos_tokens": 9, + "val/ratio": 0.991521954536438, + "val/ratio_var": 7.228642789414153e-05 + }, + { + "episode": 5536, + "epoch": 0.2795959595959596, + "eps": 0, + "loss/policy_avg": 0.31329306960105896, + "loss/value_avg": 0.5371936559677124, + "lr": 8.637886514988101e-06, + "objective/entropy": -73.7339859008789, + "objective/kl": 23.168113708496094, + "objective/non_score_reward": -1.1584056615829468, + "objective/rlhf_reward": 9.654094696044922, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.0010758393909782171, + "policy/clipfrac_avg": 0.011895336210727692, + "policy/entropy_avg": 0.1218266487121582, + "step": 346, + "val/clipfrac_avg": 0.12817031145095825, + "val/num_eos_tokens": 331, + "val/ratio": 0.9905756711959839, + "val/ratio_var": 3.3087821066146716e-05 + }, + { + "episode": 5552, + "epoch": 0.2804040404040404, + "eps": 0, + "loss/policy_avg": 0.18582414090633392, + "loss/value_avg": 0.8826534748077393, + "lr": 8.62871022309141e-06, + "objective/entropy": -90.95281219482422, + "objective/kl": 21.344974517822266, + "objective/non_score_reward": -1.067248821258545, + "objective/rlhf_reward": 11.057750701904297, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0006777847884222865, + "policy/clipfrac_avg": 0.005223071202635765, + "policy/entropy_avg": 0.12928390502929688, + "step": 347, + "val/clipfrac_avg": 0.27874505519866943, + "val/num_eos_tokens": 424, + "val/ratio": 0.9911484122276306, + "val/ratio_var": 5.034026253269985e-05 + }, + { + "episode": 5568, + "epoch": 0.2812121212121212, + "eps": 0, + "loss/policy_avg": 0.3524755835533142, + "loss/value_avg": 0.5708775520324707, + "lr": 8.619508034929646e-06, + "objective/entropy": -148.79641723632812, + "objective/kl": 24.182207107543945, + "objective/non_score_reward": -1.2091102600097656, + "objective/rlhf_reward": 6.923702239990234, + "objective/scores": 8.125, + "policy/approxkl_avg": 0.0007171371835283935, + "policy/clipfrac_avg": 0.009594540111720562, + "policy/entropy_avg": 0.12054634094238281, + "step": 348, + "val/clipfrac_avg": 0.23543015122413635, + "val/num_eos_tokens": 537, + "val/ratio": 0.9915475845336914, + "val/ratio_var": 3.1630850571673363e-05 + }, + { + "episode": 5584, + "epoch": 0.282020202020202, + "eps": 0, + "loss/policy_avg": -0.014234375208616257, + "loss/value_avg": 0.7844265103340149, + "lr": 8.610280016174157e-06, + "objective/entropy": -47.92275619506836, + "objective/kl": 25.371280670166016, + "objective/non_score_reward": -1.268563985824585, + "objective/rlhf_reward": 9.657217979431152, + "objective/scores": 10.9375, + "policy/approxkl_avg": 0.0007700816495344043, + "policy/clipfrac_avg": 0.004703640937805176, + "policy/entropy_avg": 0.140960693359375, + "step": 349, + "val/clipfrac_avg": 0.2601862847805023, + "val/num_eos_tokens": 25, + "val/ratio": 0.9903233051300049, + "val/ratio_var": 3.460677180555649e-05 + }, + { + "episode": 5600, + "epoch": 0.2828282828282828, + "eps": 0, + "loss/policy_avg": -0.09183628112077713, + "loss/value_avg": 0.18937213718891144, + "lr": 8.601026232680634e-06, + "objective/entropy": -116.39802551269531, + "objective/kl": 25.339210510253906, + "objective/non_score_reward": -1.266960620880127, + "objective/rlhf_reward": 8.072883605957031, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.000991379958577454, + "policy/clipfrac_avg": 0.006371830590069294, + "policy/entropy_avg": 0.14559483528137207, + "step": 350, + "val/clipfrac_avg": 0.04358673095703125, + "val/num_eos_tokens": 592, + "val/ratio": 0.9900184869766235, + "val/ratio_var": 3.288961306679994e-05 + }, + { + "episode": 5616, + "epoch": 0.28363636363636363, + "eps": 0, + "loss/policy_avg": -0.006830994039773941, + "loss/value_avg": 0.9599238038063049, + "lr": 8.591746750488639e-06, + "objective/entropy": -138.04812622070312, + "objective/kl": 25.083232879638672, + "objective/non_score_reward": -1.2541615962982178, + "objective/rlhf_reward": 7.921619415283203, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.0008497297530993819, + "policy/clipfrac_avg": 0.017307590693235397, + "policy/entropy_avg": 0.14270031452178955, + "step": 351, + "val/clipfrac_avg": 0.0904756635427475, + "val/num_eos_tokens": 64, + "val/ratio": 0.9911775588989258, + "val/ratio_var": 3.714514969033189e-05 + }, + { + "episode": 5632, + "epoch": 0.28444444444444444, + "eps": 0, + "loss/policy_avg": -0.29971742630004883, + "loss/value_avg": 1.0953255891799927, + "lr": 8.582441635821131e-06, + "objective/entropy": -89.55154418945312, + "objective/kl": 20.58731460571289, + "objective/non_score_reward": -1.0293657779693604, + "objective/rlhf_reward": 9.501884460449219, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.004132068250328302, + "policy/clipfrac_avg": 0.02699371986091137, + "policy/entropy_avg": 0.14103031158447266, + "step": 352, + "val/clipfrac_avg": 0.18198783695697784, + "val/num_eos_tokens": 105, + "val/ratio": 0.9924070835113525, + "val/ratio_var": 0.0006087828078307211 + }, + { + "episode": 5648, + "epoch": 0.28525252525252526, + "eps": 0, + "loss/policy_avg": 0.030918851494789124, + "loss/value_avg": 0.21527604758739471, + "lr": 8.573110955083997e-06, + "objective/entropy": -96.5112533569336, + "objective/kl": 19.544586181640625, + "objective/non_score_reward": -0.9772292971611023, + "objective/rlhf_reward": 10.460270881652832, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0007747088093310595, + "policy/clipfrac_avg": 0.0063193682581186295, + "policy/entropy_avg": 0.12494707107543945, + "step": 353, + "val/clipfrac_avg": 0.06936304271221161, + "val/num_eos_tokens": 302, + "val/ratio": 0.991653561592102, + "val/ratio_var": 4.326908310758881e-05 + }, + { + "episode": 5664, + "epoch": 0.28606060606060607, + "eps": 0, + "loss/policy_avg": 0.08415427803993225, + "loss/value_avg": 0.7643447518348694, + "lr": 8.563754774865574e-06, + "objective/entropy": -39.94751739501953, + "objective/kl": 19.486572265625, + "objective/non_score_reward": -0.9743286371231079, + "objective/rlhf_reward": 8.470983505249023, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0012616277672350407, + "policy/clipfrac_avg": 0.011073650792241096, + "policy/entropy_avg": 0.1490478515625, + "step": 354, + "val/clipfrac_avg": 0.14175766706466675, + "val/num_eos_tokens": 66, + "val/ratio": 0.9900429844856262, + "val/ratio_var": 5.867081563337706e-05 + }, + { + "episode": 5680, + "epoch": 0.2868686868686869, + "eps": 0, + "loss/policy_avg": -0.009337957948446274, + "loss/value_avg": 0.32016801834106445, + "lr": 8.554373161936176e-06, + "objective/entropy": -91.80198669433594, + "objective/kl": 21.03156852722168, + "objective/non_score_reward": -1.0515785217285156, + "objective/rlhf_reward": 8.647640228271484, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0007716022664681077, + "policy/clipfrac_avg": 0.006859911140054464, + "policy/entropy_avg": 0.15067577362060547, + "step": 355, + "val/clipfrac_avg": 0.10722831636667252, + "val/num_eos_tokens": 556, + "val/ratio": 0.9901145696640015, + "val/ratio_var": 4.587566218106076e-05 + }, + { + "episode": 5696, + "epoch": 0.2876767676767677, + "eps": 0, + "loss/policy_avg": -0.4428892731666565, + "loss/value_avg": 0.8014440536499023, + "lr": 8.544966183247618e-06, + "objective/entropy": -147.03330993652344, + "objective/kl": 26.18867301940918, + "objective/non_score_reward": -1.3094336986541748, + "objective/rlhf_reward": 8.801894187927246, + "objective/scores": 10.125, + "policy/approxkl_avg": 0.0008081612177193165, + "policy/clipfrac_avg": 0.0038806938100606203, + "policy/entropy_avg": 0.15047848224639893, + "step": 356, + "val/clipfrac_avg": 0.16974319517612457, + "val/num_eos_tokens": 353, + "val/ratio": 0.9893222451210022, + "val/ratio_var": 5.328768747858703e-05 + }, + { + "episode": 5712, + "epoch": 0.2884848484848485, + "eps": 0, + "loss/policy_avg": -0.06810799241065979, + "loss/value_avg": 0.4220166802406311, + "lr": 8.535533905932739e-06, + "objective/entropy": -75.95680236816406, + "objective/kl": 20.14461898803711, + "objective/non_score_reward": -1.0072309970855713, + "objective/rlhf_reward": 9.789644241333008, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.001012616790831089, + "policy/clipfrac_avg": 0.00791792944073677, + "policy/entropy_avg": 0.15483474731445312, + "step": 357, + "val/clipfrac_avg": 0.06293391436338425, + "val/num_eos_tokens": 70, + "val/ratio": 0.9901751279830933, + "val/ratio_var": 6.756299990229309e-05 + }, + { + "episode": 5728, + "epoch": 0.28929292929292927, + "eps": 0, + "loss/policy_avg": 0.13574761152267456, + "loss/value_avg": 1.2186156511306763, + "lr": 8.526076397304918e-06, + "objective/entropy": -78.61557006835938, + "objective/kl": 17.09016227722168, + "objective/non_score_reward": -0.8545081615447998, + "objective/rlhf_reward": 10.024397850036621, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.000848946045152843, + "policy/clipfrac_avg": 0.0072188605554401875, + "policy/entropy_avg": 0.14081084728240967, + "step": 358, + "val/clipfrac_avg": 0.13289573788642883, + "val/num_eos_tokens": 77, + "val/ratio": 0.9911394715309143, + "val/ratio_var": 5.175899059395306e-05 + }, + { + "episode": 5744, + "epoch": 0.2901010101010101, + "eps": 0, + "loss/policy_avg": 0.8527805805206299, + "loss/value_avg": 1.6828011274337769, + "lr": 8.516593724857598e-06, + "objective/entropy": -91.85313415527344, + "objective/kl": 18.7655029296875, + "objective/non_score_reward": -0.938275158405304, + "objective/rlhf_reward": 7.577349662780762, + "objective/scores": 8.5, + "policy/approxkl_avg": 0.0012549037346616387, + "policy/clipfrac_avg": 0.013728092424571514, + "policy/entropy_avg": 0.17795109748840332, + "step": 359, + "val/clipfrac_avg": 0.40935200452804565, + "val/num_eos_tokens": 160, + "val/ratio": 0.9902350902557373, + "val/ratio_var": 4.9665297410683706e-05 + }, + { + "episode": 5760, + "epoch": 0.2909090909090909, + "eps": 0, + "loss/policy_avg": 0.38345038890838623, + "loss/value_avg": 1.2663400173187256, + "lr": 8.507085956263808e-06, + "objective/entropy": -113.52433013916016, + "objective/kl": 19.024890899658203, + "objective/non_score_reward": -0.9512445330619812, + "objective/rlhf_reward": 10.044849395751953, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010559242218732834, + "policy/clipfrac_avg": 0.011119586415588856, + "policy/entropy_avg": 0.15532588958740234, + "step": 360, + "val/clipfrac_avg": 0.20008543133735657, + "val/num_eos_tokens": 110, + "val/ratio": 0.9906458854675293, + "val/ratio_var": 5.071852865512483e-05 + }, + { + "episode": 5776, + "epoch": 0.2917171717171717, + "eps": 0, + "loss/policy_avg": 0.0799669474363327, + "loss/value_avg": 0.7555652856826782, + "lr": 8.49755315937567e-06, + "objective/entropy": -220.52227783203125, + "objective/kl": 22.22720718383789, + "objective/non_score_reward": -1.1113603115081787, + "objective/rlhf_reward": 8.072233200073242, + "objective/scores": 9.1875, + "policy/approxkl_avg": 0.000943245948292315, + "policy/clipfrac_avg": 0.01834721677005291, + "policy/entropy_avg": 0.1559591293334961, + "step": 361, + "val/clipfrac_avg": 0.11736905574798584, + "val/num_eos_tokens": 117, + "val/ratio": 0.990176796913147, + "val/ratio_var": 7.675655069760978e-05 + }, + { + "episode": 5792, + "epoch": 0.2925252525252525, + "eps": 0, + "loss/policy_avg": 0.24114887416362762, + "loss/value_avg": 2.6182055473327637, + "lr": 8.487995402223922e-06, + "objective/entropy": -127.6094970703125, + "objective/kl": 19.329463958740234, + "objective/non_score_reward": -0.9664731621742249, + "objective/rlhf_reward": 8.408527374267578, + "objective/scores": 9.375, + "policy/approxkl_avg": 0.0006040905136615038, + "policy/clipfrac_avg": 0.004784829448908567, + "policy/entropy_avg": 0.12860214710235596, + "step": 362, + "val/clipfrac_avg": 0.27273598313331604, + "val/num_eos_tokens": 598, + "val/ratio": 0.9914668202400208, + "val/ratio_var": 5.468882591230795e-05 + }, + { + "episode": 5808, + "epoch": 0.29333333333333333, + "eps": 0, + "loss/policy_avg": 0.25809526443481445, + "loss/value_avg": 0.4706522822380066, + "lr": 8.478412753017433e-06, + "objective/entropy": -125.06207275390625, + "objective/kl": 24.048336029052734, + "objective/non_score_reward": -1.2024168968200684, + "objective/rlhf_reward": 8.113988876342773, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0009443694725632668, + "policy/clipfrac_avg": 0.013166669756174088, + "policy/entropy_avg": 0.14909744262695312, + "step": 363, + "val/clipfrac_avg": 0.10755910724401474, + "val/num_eos_tokens": 628, + "val/ratio": 0.990389347076416, + "val/ratio_var": 2.328440132259857e-05 + }, + { + "episode": 5824, + "epoch": 0.29414141414141415, + "eps": 0, + "loss/policy_avg": 0.15223711729049683, + "loss/value_avg": 1.0644316673278809, + "lr": 8.46880528014271e-06, + "objective/entropy": -92.4540786743164, + "objective/kl": 28.906822204589844, + "objective/non_score_reward": -1.4453411102294922, + "objective/rlhf_reward": 8.749971389770508, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0010854683350771666, + "policy/clipfrac_avg": 0.027407623827457428, + "policy/entropy_avg": 0.18361949920654297, + "step": 364, + "val/clipfrac_avg": 0.15570449829101562, + "val/num_eos_tokens": 6, + "val/ratio": 0.9883888959884644, + "val/ratio_var": 3.197947080479935e-05 + }, + { + "episode": 5840, + "epoch": 0.29494949494949496, + "eps": 0, + "loss/policy_avg": 0.0015800148248672485, + "loss/value_avg": 0.2736946940422058, + "lr": 8.459173052163414e-06, + "objective/entropy": -123.84210205078125, + "objective/kl": 27.184215545654297, + "objective/non_score_reward": -1.359210729598999, + "objective/rlhf_reward": 7.44547700881958, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0008880237000994384, + "policy/clipfrac_avg": 0.004617786966264248, + "policy/entropy_avg": 0.16141414642333984, + "step": 365, + "val/clipfrac_avg": 0.1854676902294159, + "val/num_eos_tokens": 360, + "val/ratio": 0.9891216158866882, + "val/ratio_var": 4.489487400860526e-05 + }, + { + "episode": 5856, + "epoch": 0.2957575757575758, + "eps": 0, + "loss/policy_avg": 0.012775429524481297, + "loss/value_avg": 0.48028677701950073, + "lr": 8.449516137819875e-06, + "objective/entropy": -79.13996887207031, + "objective/kl": 24.248794555664062, + "objective/non_score_reward": -1.212439775466919, + "objective/rlhf_reward": 7.16060733795166, + "objective/scores": 8.375, + "policy/approxkl_avg": 0.0008022888214327395, + "policy/clipfrac_avg": 0.006077153142541647, + "policy/entropy_avg": 0.16005897521972656, + "step": 366, + "val/clipfrac_avg": 0.04264098405838013, + "val/num_eos_tokens": 176, + "val/ratio": 0.9902619123458862, + "val/ratio_var": 3.514832860673778e-05 + }, + { + "episode": 5872, + "epoch": 0.2965656565656566, + "eps": 0, + "loss/policy_avg": -0.18061695992946625, + "loss/value_avg": 0.539659857749939, + "lr": 8.439834606028594e-06, + "objective/entropy": -266.6590270996094, + "objective/kl": 32.35683822631836, + "objective/non_score_reward": -1.6178419589996338, + "objective/rlhf_reward": 8.089189529418945, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006289628800004721, + "policy/clipfrac_avg": 0.009195220656692982, + "policy/entropy_avg": 0.1150360107421875, + "step": 367, + "val/clipfrac_avg": 0.10509515553712845, + "val/num_eos_tokens": 414, + "val/ratio": 0.9929022789001465, + "val/ratio_var": 3.284734339104034e-05 + }, + { + "episode": 5888, + "epoch": 0.2973737373737374, + "eps": 0, + "loss/policy_avg": -0.16081279516220093, + "loss/value_avg": 1.9553475379943848, + "lr": 8.430128525881756e-06, + "objective/entropy": -54.5378532409668, + "objective/kl": 26.481590270996094, + "objective/non_score_reward": -1.3240795135498047, + "objective/rlhf_reward": 11.027482986450195, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.001065750140696764, + "policy/clipfrac_avg": 0.006770070642232895, + "policy/entropy_avg": 0.18307781219482422, + "step": 368, + "val/clipfrac_avg": 0.1946491301059723, + "val/num_eos_tokens": 329, + "val/ratio": 0.9881594777107239, + "val/ratio_var": 5.794196476927027e-05 + }, + { + "episode": 5904, + "epoch": 0.29818181818181816, + "eps": 0, + "loss/policy_avg": -0.06483446806669235, + "loss/value_avg": 0.38115546107292175, + "lr": 8.420397966646732e-06, + "objective/entropy": -171.61798095703125, + "objective/kl": 27.11490249633789, + "objective/non_score_reward": -1.3557450771331787, + "objective/rlhf_reward": 7.8825364112854, + "objective/scores": 9.25, + "policy/approxkl_avg": 0.0009488058858551085, + "policy/clipfrac_avg": 0.008429555222392082, + "policy/entropy_avg": 0.17403793334960938, + "step": 369, + "val/clipfrac_avg": 0.09484779834747314, + "val/num_eos_tokens": 417, + "val/ratio": 0.9889323711395264, + "val/ratio_var": 5.186999624129385e-05 + }, + { + "episode": 5920, + "epoch": 0.298989898989899, + "eps": 0, + "loss/policy_avg": -0.01815527118742466, + "loss/value_avg": 0.25550493597984314, + "lr": 8.41064299776559e-06, + "objective/entropy": -61.57228088378906, + "objective/kl": 25.170589447021484, + "objective/non_score_reward": -1.2585294246673584, + "objective/rlhf_reward": 7.518814563751221, + "objective/scores": 8.75, + "policy/approxkl_avg": 0.001179067767225206, + "policy/clipfrac_avg": 0.0051048677414655685, + "policy/entropy_avg": 0.15404510498046875, + "step": 370, + "val/clipfrac_avg": 0.060221657156944275, + "val/num_eos_tokens": 108, + "val/ratio": 0.9892715215682983, + "val/ratio_var": 3.491001552902162e-05 + }, + { + "episode": 5936, + "epoch": 0.2997979797979798, + "eps": 0, + "loss/policy_avg": -0.10499698668718338, + "loss/value_avg": 0.6281431317329407, + "lr": 8.400863688854598e-06, + "objective/entropy": -62.79341125488281, + "objective/kl": 25.240398406982422, + "objective/non_score_reward": -1.2620198726654053, + "objective/rlhf_reward": 9.948917388916016, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0009268317371606827, + "policy/clipfrac_avg": 0.021911144256591797, + "policy/entropy_avg": 0.15642356872558594, + "step": 371, + "val/clipfrac_avg": 0.10369566828012466, + "val/num_eos_tokens": 82, + "val/ratio": 0.989446759223938, + "val/ratio_var": 4.36611044278834e-05 + }, + { + "episode": 5952, + "epoch": 0.3006060606060606, + "eps": 0, + "loss/policy_avg": -0.3728405237197876, + "loss/value_avg": 1.200590968132019, + "lr": 8.391060109703725e-06, + "objective/entropy": -90.6949234008789, + "objective/kl": 24.834531784057617, + "objective/non_score_reward": -1.2417266368865967, + "objective/rlhf_reward": 8.82468032836914, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001307523576542735, + "policy/clipfrac_avg": 0.011635184288024902, + "policy/entropy_avg": 0.14268022775650024, + "step": 372, + "val/clipfrac_avg": 0.09476368129253387, + "val/num_eos_tokens": 348, + "val/ratio": 0.9903372526168823, + "val/ratio_var": 5.383575626183301e-05 + }, + { + "episode": 5968, + "epoch": 0.3014141414141414, + "eps": 0, + "loss/policy_avg": -0.2683873772621155, + "loss/value_avg": 0.8197465538978577, + "lr": 8.381232330276145e-06, + "objective/entropy": -77.07698822021484, + "objective/kl": 23.921588897705078, + "objective/non_score_reward": -1.1960794925689697, + "objective/rlhf_reward": 8.25704574584961, + "objective/scores": 9.4375, + "policy/approxkl_avg": 0.0010654187062755227, + "policy/clipfrac_avg": 0.013933488167822361, + "policy/entropy_avg": 0.16466379165649414, + "step": 373, + "val/clipfrac_avg": 0.11054794490337372, + "val/num_eos_tokens": 452, + "val/ratio": 0.9914400577545166, + "val/ratio_var": 7.72410276113078e-05 + }, + { + "episode": 5984, + "epoch": 0.3022222222222222, + "eps": 0, + "loss/policy_avg": 0.050479210913181305, + "loss/value_avg": 0.2513907849788666, + "lr": 8.371380420707737e-06, + "objective/entropy": -90.79975891113281, + "objective/kl": 20.35816192626953, + "objective/non_score_reward": -1.0179080963134766, + "objective/rlhf_reward": 9.146154403686523, + "objective/scores": 10.1875, + "policy/approxkl_avg": 0.0011633289977908134, + "policy/clipfrac_avg": 0.011328942142426968, + "policy/entropy_avg": 0.15205001831054688, + "step": 374, + "val/clipfrac_avg": 0.0933825820684433, + "val/num_eos_tokens": 292, + "val/ratio": 0.98993980884552, + "val/ratio_var": 3.12502306769602e-05 + }, + { + "episode": 6000, + "epoch": 0.30303030303030304, + "eps": 0, + "loss/policy_avg": -0.24270930886268616, + "loss/value_avg": 1.0323677062988281, + "lr": 8.361504451306585e-06, + "objective/entropy": -152.4559326171875, + "objective/kl": 19.31946563720703, + "objective/non_score_reward": -0.9659732580184937, + "objective/rlhf_reward": 10.717620849609375, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0006288137519732118, + "policy/clipfrac_avg": 0.0182405486702919, + "policy/entropy_avg": 0.14447784423828125, + "step": 375, + "val/clipfrac_avg": 0.2301131635904312, + "val/num_eos_tokens": 572, + "val/ratio": 0.9926867485046387, + "val/ratio_var": 3.9746519178152084e-05 + }, + { + "episode": 6016, + "epoch": 0.30383838383838385, + "eps": 0, + "loss/policy_avg": -0.6204806566238403, + "loss/value_avg": 0.6857134103775024, + "lr": 8.351604492552473e-06, + "objective/entropy": -204.8240966796875, + "objective/kl": 19.163963317871094, + "objective/non_score_reward": -0.9581981897354126, + "objective/rlhf_reward": 10.190238952636719, + "objective/scores": 11.125, + "policy/approxkl_avg": 0.000608496367931366, + "policy/clipfrac_avg": 0.005058623850345612, + "policy/entropy_avg": 0.1286005973815918, + "step": 376, + "val/clipfrac_avg": 0.1522846519947052, + "val/num_eos_tokens": 235, + "val/ratio": 0.9927879571914673, + "val/ratio_var": 4.832358536077663e-05 + }, + { + "episode": 6032, + "epoch": 0.30464646464646467, + "eps": 0, + "loss/policy_avg": -0.058100730180740356, + "loss/value_avg": 0.4977622628211975, + "lr": 8.341680615096392e-06, + "objective/entropy": -81.702392578125, + "objective/kl": 20.53563690185547, + "objective/non_score_reward": -1.0267819166183472, + "objective/rlhf_reward": 9.441967964172363, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0008746698149479926, + "policy/clipfrac_avg": 0.005690821446478367, + "policy/entropy_avg": 0.1544971466064453, + "step": 377, + "val/clipfrac_avg": 0.22529853880405426, + "val/num_eos_tokens": 62, + "val/ratio": 0.9891985654830933, + "val/ratio_var": 4.927700138068758e-05 + }, + { + "episode": 6048, + "epoch": 0.3054545454545455, + "eps": 0, + "loss/policy_avg": -0.5821992754936218, + "loss/value_avg": 1.1654460430145264, + "lr": 8.331732889760021e-06, + "objective/entropy": -140.73626708984375, + "objective/kl": 19.327545166015625, + "objective/non_score_reward": -0.966377317905426, + "objective/rlhf_reward": 10.248466491699219, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.0006984242936596274, + "policy/clipfrac_avg": 0.004717235453426838, + "policy/entropy_avg": 0.12701988220214844, + "step": 378, + "val/clipfrac_avg": 0.29272496700286865, + "val/num_eos_tokens": 51, + "val/ratio": 0.9922663569450378, + "val/ratio_var": 5.01677677675616e-05 + }, + { + "episode": 6064, + "epoch": 0.30626262626262624, + "eps": 0, + "loss/policy_avg": 0.37417128682136536, + "loss/value_avg": 1.0575264692306519, + "lr": 8.321761387535231e-06, + "objective/entropy": -77.88591766357422, + "objective/kl": 20.278968811035156, + "objective/non_score_reward": -1.0139484405517578, + "objective/rlhf_reward": 10.259489059448242, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008548812475055456, + "policy/clipfrac_avg": 0.00871540792286396, + "policy/entropy_avg": 0.14256516098976135, + "step": 379, + "val/clipfrac_avg": 0.27003729343414307, + "val/num_eos_tokens": 68, + "val/ratio": 0.9905642867088318, + "val/ratio_var": 4.364799315226264e-05 + }, + { + "episode": 6080, + "epoch": 0.30707070707070705, + "eps": 0, + "loss/policy_avg": 0.1371518224477768, + "loss/value_avg": 0.33893001079559326, + "lr": 8.31176617958358e-06, + "objective/entropy": -30.368871688842773, + "objective/kl": 16.528039932250977, + "objective/non_score_reward": -0.826401948928833, + "objective/rlhf_reward": 8.12281608581543, + "objective/scores": 8.9375, + "policy/approxkl_avg": 0.0010957561898976564, + "policy/clipfrac_avg": 0.008557099848985672, + "policy/entropy_avg": 0.14247894287109375, + "step": 380, + "val/clipfrac_avg": 0.09286786615848541, + "val/num_eos_tokens": 117, + "val/ratio": 0.9898958802223206, + "val/ratio_var": 2.362048689974472e-05 + }, + { + "episode": 6096, + "epoch": 0.30787878787878786, + "eps": 0, + "loss/policy_avg": 0.09494100511074066, + "loss/value_avg": 0.45570412278175354, + "lr": 8.301747337235798e-06, + "objective/entropy": -30.008560180664062, + "objective/kl": 23.604557037353516, + "objective/non_score_reward": -1.1802278757095337, + "objective/rlhf_reward": 8.143990516662598, + "objective/scores": 9.3125, + "policy/approxkl_avg": 0.0010318540735170245, + "policy/clipfrac_avg": 0.008254901506006718, + "policy/entropy_avg": 0.18322467803955078, + "step": 381, + "val/clipfrac_avg": 0.060982413589954376, + "val/num_eos_tokens": 592, + "val/ratio": 0.988663375377655, + "val/ratio_var": 3.0003804567968473e-05 + }, + { + "episode": 6112, + "epoch": 0.3086868686868687, + "eps": 0, + "loss/policy_avg": -0.1134948879480362, + "loss/value_avg": 0.7391153573989868, + "lr": 8.291704931991283e-06, + "objective/entropy": -116.08049011230469, + "objective/kl": 21.36620330810547, + "objective/non_score_reward": -1.068310022354126, + "objective/rlhf_reward": 10.908252716064453, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010032784193754196, + "policy/clipfrac_avg": 0.007687012664973736, + "policy/entropy_avg": 0.13149738311767578, + "step": 382, + "val/clipfrac_avg": 0.11116228997707367, + "val/num_eos_tokens": 273, + "val/ratio": 0.9909749627113342, + "val/ratio_var": 3.5987435694551095e-05 + }, + { + "episode": 6128, + "epoch": 0.3094949494949495, + "eps": 0, + "loss/policy_avg": 0.26366716623306274, + "loss/value_avg": 0.31152695417404175, + "lr": 8.281639035517591e-06, + "objective/entropy": -64.46329498291016, + "objective/kl": 24.368152618408203, + "objective/non_score_reward": -1.2184076309204102, + "objective/rlhf_reward": 7.60581111907959, + "objective/scores": 8.8125, + "policy/approxkl_avg": 0.0010871912818402052, + "policy/clipfrac_avg": 0.009455868974328041, + "policy/entropy_avg": 0.20527267456054688, + "step": 383, + "val/clipfrac_avg": 0.17807866632938385, + "val/num_eos_tokens": 14, + "val/ratio": 0.9889589548110962, + "val/ratio_var": 4.2642153857741505e-05 + }, + { + "episode": 6144, + "epoch": 0.3103030303030303, + "eps": 0, + "loss/policy_avg": -0.2564522624015808, + "loss/value_avg": 1.2739346027374268, + "lr": 8.271549719649923e-06, + "objective/entropy": -112.77015686035156, + "objective/kl": 23.171056747436523, + "objective/non_score_reward": -1.158552885055542, + "objective/rlhf_reward": 11.263322830200195, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009117453591898084, + "policy/clipfrac_avg": 0.0171449463814497, + "policy/entropy_avg": 0.1448192596435547, + "step": 384, + "val/clipfrac_avg": 0.2111937403678894, + "val/num_eos_tokens": 329, + "val/ratio": 0.9912400245666504, + "val/ratio_var": 3.967702286900021e-05 + }, + { + "episode": 6160, + "epoch": 0.3111111111111111, + "eps": 0, + "loss/policy_avg": 0.22879835963249207, + "loss/value_avg": 0.6531267166137695, + "lr": 8.261437056390607e-06, + "objective/entropy": -142.6223602294922, + "objective/kl": 22.004093170166016, + "objective/non_score_reward": -1.1002047061920166, + "objective/rlhf_reward": 8.985733032226562, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.001018032431602478, + "policy/clipfrac_avg": 0.009698743000626564, + "policy/entropy_avg": 0.15434914827346802, + "step": 385, + "val/clipfrac_avg": 0.10454542934894562, + "val/num_eos_tokens": 182, + "val/ratio": 0.9905833601951599, + "val/ratio_var": 8.004864503163844e-05 + }, + { + "episode": 6176, + "epoch": 0.31191919191919193, + "eps": 0, + "loss/policy_avg": 0.1902710199356079, + "loss/value_avg": 0.6345528960227966, + "lr": 8.251301117908595e-06, + "objective/entropy": -98.07450866699219, + "objective/kl": 24.104801177978516, + "objective/non_score_reward": -1.20524001121521, + "objective/rlhf_reward": 9.251791000366211, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008569885976612568, + "policy/clipfrac_avg": 0.009619549848139286, + "policy/entropy_avg": 0.15359973907470703, + "step": 386, + "val/clipfrac_avg": 0.04851880669593811, + "val/num_eos_tokens": 67, + "val/ratio": 0.9899508357048035, + "val/ratio_var": 4.097054261364974e-05 + }, + { + "episode": 6192, + "epoch": 0.31272727272727274, + "eps": 0, + "loss/policy_avg": 0.17917793989181519, + "loss/value_avg": 0.36408013105392456, + "lr": 8.241141976538944e-06, + "objective/entropy": -110.4149398803711, + "objective/kl": 22.46795654296875, + "objective/non_score_reward": -1.1233978271484375, + "objective/rlhf_reward": 8.353164672851562, + "objective/scores": 9.5, + "policy/approxkl_avg": 0.0009543322958052158, + "policy/clipfrac_avg": 0.014605259522795677, + "policy/entropy_avg": 0.16858911514282227, + "step": 387, + "val/clipfrac_avg": 0.16161151230335236, + "val/num_eos_tokens": 273, + "val/ratio": 0.9885615706443787, + "val/ratio_var": 4.8216566938208416e-05 + }, + { + "episode": 6208, + "epoch": 0.31353535353535356, + "eps": 0, + "loss/policy_avg": -0.27181899547576904, + "loss/value_avg": 1.2867136001586914, + "lr": 8.230959704782289e-06, + "objective/entropy": -77.30952453613281, + "objective/kl": 19.66629409790039, + "objective/non_score_reward": -0.9833146333694458, + "objective/rlhf_reward": 10.782310485839844, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008605445618741214, + "policy/clipfrac_avg": 0.010454193688929081, + "policy/entropy_avg": 0.1627795696258545, + "step": 388, + "val/clipfrac_avg": 0.2507970631122589, + "val/num_eos_tokens": 135, + "val/ratio": 0.9896144866943359, + "val/ratio_var": 5.392284947447479e-05 + }, + { + "episode": 6224, + "epoch": 0.31434343434343437, + "eps": 0, + "loss/policy_avg": -0.24307014048099518, + "loss/value_avg": 0.3495979309082031, + "lr": 8.220754375304348e-06, + "objective/entropy": -50.25094985961914, + "objective/kl": 24.12609100341797, + "objective/non_score_reward": -1.2063045501708984, + "objective/rlhf_reward": 8.664789199829102, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0011184238828718662, + "policy/clipfrac_avg": 0.013984918594360352, + "policy/entropy_avg": 0.1985645294189453, + "step": 389, + "val/clipfrac_avg": 0.06929454207420349, + "val/num_eos_tokens": 188, + "val/ratio": 0.9888651371002197, + "val/ratio_var": 4.258898115949705e-05 + }, + { + "episode": 6240, + "epoch": 0.3151515151515151, + "eps": 0, + "loss/policy_avg": -0.4267215132713318, + "loss/value_avg": 0.9314700365066528, + "lr": 8.210526060935377e-06, + "objective/entropy": -112.65766906738281, + "objective/kl": 21.656740188598633, + "objective/non_score_reward": -1.0828369855880737, + "objective/rlhf_reward": 9.667162895202637, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.000986731261946261, + "policy/clipfrac_avg": 0.006748675834387541, + "policy/entropy_avg": 0.19133377075195312, + "step": 390, + "val/clipfrac_avg": 0.14616435766220093, + "val/num_eos_tokens": 10, + "val/ratio": 0.988615870475769, + "val/ratio_var": 5.443965710583143e-05 + }, + { + "episode": 6256, + "epoch": 0.31595959595959594, + "eps": 0, + "loss/policy_avg": -0.27975156903266907, + "loss/value_avg": 0.42042723298072815, + "lr": 8.200274834669675e-06, + "objective/entropy": -93.58556365966797, + "objective/kl": 18.75835609436035, + "objective/non_score_reward": -0.9379178285598755, + "objective/rlhf_reward": 10.554269790649414, + "objective/scores": 11.5, + "policy/approxkl_avg": 0.0006623042281717062, + "policy/clipfrac_avg": 0.010373012162744999, + "policy/entropy_avg": 0.1245126724243164, + "step": 391, + "val/clipfrac_avg": 0.10715407133102417, + "val/num_eos_tokens": 243, + "val/ratio": 0.9925373792648315, + "val/ratio_var": 3.973143975599669e-05 + }, + { + "episode": 6272, + "epoch": 0.31676767676767675, + "eps": 0, + "loss/policy_avg": -0.20366400480270386, + "loss/value_avg": 0.8359952569007874, + "lr": 8.190000769665044e-06, + "objective/entropy": -50.36949920654297, + "objective/kl": 22.554901123046875, + "objective/non_score_reward": -1.1277451515197754, + "objective/rlhf_reward": 10.442567825317383, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0012943993788212538, + "policy/clipfrac_avg": 0.01801796816289425, + "policy/entropy_avg": 0.17497825622558594, + "step": 392, + "val/clipfrac_avg": 0.15647155046463013, + "val/num_eos_tokens": 535, + "val/ratio": 0.9895493984222412, + "val/ratio_var": 0.00010072851000586525 + }, + { + "episode": 6288, + "epoch": 0.31757575757575757, + "eps": 0, + "loss/policy_avg": -0.2513708472251892, + "loss/value_avg": 0.5702548623085022, + "lr": 8.179703939242276e-06, + "objective/entropy": -77.18543243408203, + "objective/kl": 23.501121520996094, + "objective/non_score_reward": -1.175055980682373, + "objective/rlhf_reward": 9.567131042480469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0009706417913548648, + "policy/clipfrac_avg": 0.016374697908759117, + "policy/entropy_avg": 0.16623497009277344, + "step": 393, + "val/clipfrac_avg": 0.16020084917545319, + "val/num_eos_tokens": 31, + "val/ratio": 0.9895526170730591, + "val/ratio_var": 4.693511436926201e-05 + }, + { + "episode": 6304, + "epoch": 0.3183838383838384, + "eps": 0, + "loss/policy_avg": -0.2903949022293091, + "loss/value_avg": 0.7161914110183716, + "lr": 8.169384416884632e-06, + "objective/entropy": -76.87279510498047, + "objective/kl": 24.797298431396484, + "objective/non_score_reward": -1.2398649454116821, + "objective/rlhf_reward": 10.158573150634766, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0008667815709486604, + "policy/clipfrac_avg": 0.007015440613031387, + "policy/entropy_avg": 0.15674304962158203, + "step": 394, + "val/clipfrac_avg": 0.066227987408638, + "val/num_eos_tokens": 344, + "val/ratio": 0.9892519116401672, + "val/ratio_var": 6.106436921982095e-05 + }, + { + "episode": 6320, + "epoch": 0.3191919191919192, + "eps": 0, + "loss/policy_avg": -0.030849093571305275, + "loss/value_avg": 0.2917250990867615, + "lr": 8.159042276237308e-06, + "objective/entropy": -162.0740966796875, + "objective/kl": 21.726106643676758, + "objective/non_score_reward": -1.0863052606582642, + "objective/rlhf_reward": 8.612913131713867, + "objective/scores": 9.6875, + "policy/approxkl_avg": 0.0006859052227810025, + "policy/clipfrac_avg": 0.0042783659882843494, + "policy/entropy_avg": 0.13499069213867188, + "step": 395, + "val/clipfrac_avg": 0.08182407170534134, + "val/num_eos_tokens": 190, + "val/ratio": 0.9912937879562378, + "val/ratio_var": 2.8321417630650103e-05 + }, + { + "episode": 6336, + "epoch": 0.32, + "eps": 0, + "loss/policy_avg": -0.13514100015163422, + "loss/value_avg": 2.264462947845459, + "lr": 8.148677591106919e-06, + "objective/entropy": -90.22354125976562, + "objective/kl": 19.762617111206055, + "objective/non_score_reward": -0.9881308674812317, + "objective/rlhf_reward": 10.918119430541992, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008516041561961174, + "policy/clipfrac_avg": 0.008233188651502132, + "policy/entropy_avg": 0.1468067169189453, + "step": 396, + "val/clipfrac_avg": 0.27357757091522217, + "val/num_eos_tokens": 119, + "val/ratio": 0.9900863766670227, + "val/ratio_var": 6.002257941872813e-05 + }, + { + "episode": 6352, + "epoch": 0.3208080808080808, + "eps": 0, + "loss/policy_avg": -0.5209656357765198, + "loss/value_avg": 0.5755888819694519, + "lr": 8.138290435460969e-06, + "objective/entropy": -150.10548400878906, + "objective/kl": 21.18748664855957, + "objective/non_score_reward": -1.0593743324279785, + "objective/rlhf_reward": 10.34687614440918, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000792744685895741, + "policy/clipfrac_avg": 0.017449360340833664, + "policy/entropy_avg": 0.1412447690963745, + "step": 397, + "val/clipfrac_avg": 0.11896035820245743, + "val/num_eos_tokens": 527, + "val/ratio": 0.9914792776107788, + "val/ratio_var": 5.1306455134181306e-05 + }, + { + "episode": 6368, + "epoch": 0.32161616161616163, + "eps": 0, + "loss/policy_avg": -0.40576979517936707, + "loss/value_avg": 0.33822232484817505, + "lr": 8.12788088342732e-06, + "objective/entropy": -149.72897338867188, + "objective/kl": 22.376163482666016, + "objective/non_score_reward": -1.1188081502914429, + "objective/rlhf_reward": 10.240567207336426, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0007364015327766538, + "policy/clipfrac_avg": 0.008705372922122478, + "policy/entropy_avg": 0.13206863403320312, + "step": 398, + "val/clipfrac_avg": 0.09983381628990173, + "val/num_eos_tokens": 92, + "val/ratio": 0.9914937019348145, + "val/ratio_var": 3.097344233538024e-05 + }, + { + "episode": 6384, + "epoch": 0.32242424242424245, + "eps": 0, + "loss/policy_avg": -0.29814594984054565, + "loss/value_avg": 0.8969987034797668, + "lr": 8.117449009293668e-06, + "objective/entropy": -94.70143127441406, + "objective/kl": 18.716007232666016, + "objective/non_score_reward": -0.9358003735542297, + "objective/rlhf_reward": 9.927480697631836, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008329761330969632, + "policy/clipfrac_avg": 0.010793356224894524, + "policy/entropy_avg": 0.1497507095336914, + "step": 399, + "val/clipfrac_avg": 0.11709762364625931, + "val/num_eos_tokens": 115, + "val/ratio": 0.990269660949707, + "val/ratio_var": 4.7837544116191566e-05 + }, + { + "episode": 6400, + "epoch": 0.32323232323232326, + "eps": 0, + "loss/policy_avg": -1.01192045211792, + "loss/value_avg": 1.0538803339004517, + "lr": 8.10699488750701e-06, + "objective/entropy": -178.26353454589844, + "objective/kl": 21.73818588256836, + "objective/non_score_reward": -1.0869094133377075, + "objective/rlhf_reward": 10.709965705871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0007762957829982042, + "policy/clipfrac_avg": 0.004946828819811344, + "policy/entropy_avg": 0.1459665298461914, + "step": 400, + "val/clipfrac_avg": 0.1204109638929367, + "val/num_eos_tokens": 105, + "val/ratio": 0.9914135932922363, + "val/ratio_var": 3.936345819965936e-05 + }, + { + "episode": 6416, + "epoch": 0.324040404040404, + "eps": 0, + "loss/policy_avg": 0.007637389004230499, + "loss/value_avg": 0.6951870918273926, + "lr": 8.096518592673113e-06, + "objective/entropy": -49.77229309082031, + "objective/kl": 22.773834228515625, + "objective/non_score_reward": -1.138691782951355, + "objective/rlhf_reward": 8.751934051513672, + "objective/scores": 9.875, + "policy/approxkl_avg": 0.0009881152072921395, + "policy/clipfrac_avg": 0.006775183603167534, + "policy/entropy_avg": 0.18160057067871094, + "step": 401, + "val/clipfrac_avg": 0.1905401647090912, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880608320236206, + "val/ratio_var": 3.765714427572675e-05 + }, + { + "episode": 6432, + "epoch": 0.32484848484848483, + "eps": 0, + "loss/policy_avg": -0.21244871616363525, + "loss/value_avg": 0.5243785381317139, + "lr": 8.08602019955598e-06, + "objective/entropy": -94.79197692871094, + "objective/kl": 22.062274932861328, + "objective/non_score_reward": -1.1031136512756348, + "objective/rlhf_reward": 11.678136825561523, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0007612872868776321, + "policy/clipfrac_avg": 0.008526662364602089, + "policy/entropy_avg": 0.16756629943847656, + "step": 402, + "val/clipfrac_avg": 0.25014567375183105, + "val/num_eos_tokens": 163, + "val/ratio": 0.9909036159515381, + "val/ratio_var": 3.402508809813298e-05 + }, + { + "episode": 6448, + "epoch": 0.32565656565656564, + "eps": 0, + "loss/policy_avg": 0.06411070376634598, + "loss/value_avg": 0.7002878189086914, + "lr": 8.075499783077321e-06, + "objective/entropy": -119.76268005371094, + "objective/kl": 18.592615127563477, + "objective/non_score_reward": -0.9296307563781738, + "objective/rlhf_reward": 9.375057220458984, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.0009499984444119036, + "policy/clipfrac_avg": 0.00850006751716137, + "policy/entropy_avg": 0.1573188304901123, + "step": 403, + "val/clipfrac_avg": 0.06237310171127319, + "val/num_eos_tokens": 601, + "val/ratio": 0.9920597076416016, + "val/ratio_var": 3.0845607398077846e-05 + }, + { + "episode": 6464, + "epoch": 0.32646464646464646, + "eps": 0, + "loss/policy_avg": -0.07570268213748932, + "loss/value_avg": 0.5015542507171631, + "lr": 8.064957418316014e-06, + "objective/entropy": -34.34098434448242, + "objective/kl": 20.830650329589844, + "objective/non_score_reward": -1.0415325164794922, + "objective/rlhf_reward": 8.931123733520508, + "objective/scores": 10.0, + "policy/approxkl_avg": 0.001183489803224802, + "policy/clipfrac_avg": 0.011174894869327545, + "policy/entropy_avg": 0.18660354614257812, + "step": 404, + "val/clipfrac_avg": 0.081820048391819, + "val/num_eos_tokens": 52, + "val/ratio": 0.9876670837402344, + "val/ratio_var": 4.6252916945377365e-05 + }, + { + "episode": 6480, + "epoch": 0.32727272727272727, + "eps": 0, + "loss/policy_avg": 0.2763940095901489, + "loss/value_avg": 0.7284906506538391, + "lr": 8.054393180507572e-06, + "objective/entropy": -124.3838119506836, + "objective/kl": 23.338232040405273, + "objective/non_score_reward": -1.1669116020202637, + "objective/rlhf_reward": 8.567462921142578, + "objective/scores": 9.75, + "policy/approxkl_avg": 0.0008951124618761241, + "policy/clipfrac_avg": 0.017911501228809357, + "policy/entropy_avg": 0.14433574676513672, + "step": 405, + "val/clipfrac_avg": 0.062357835471630096, + "val/num_eos_tokens": 413, + "val/ratio": 0.990904688835144, + "val/ratio_var": 2.749706618487835e-05 + }, + { + "episode": 6496, + "epoch": 0.3280808080808081, + "eps": 0, + "loss/policy_avg": -0.36677244305610657, + "loss/value_avg": 0.46131956577301025, + "lr": 8.043807145043604e-06, + "objective/entropy": -124.5007095336914, + "objective/kl": 23.43341064453125, + "objective/non_score_reward": -1.1716704368591309, + "objective/rlhf_reward": 8.894735336303711, + "objective/scores": 10.0625, + "policy/approxkl_avg": 0.0008479490061290562, + "policy/clipfrac_avg": 0.013162427581846714, + "policy/entropy_avg": 0.19167137145996094, + "step": 406, + "val/clipfrac_avg": 0.1444421261548996, + "val/num_eos_tokens": 311, + "val/ratio": 0.9900524020195007, + "val/ratio_var": 6.206807302078232e-05 + }, + { + "episode": 6512, + "epoch": 0.3288888888888889, + "eps": 0, + "loss/policy_avg": 0.04725009202957153, + "loss/value_avg": 1.2189075946807861, + "lr": 8.033199387471278e-06, + "objective/entropy": -91.82843017578125, + "objective/kl": 19.725975036621094, + "objective/non_score_reward": -0.9862988591194153, + "objective/rlhf_reward": 10.459013938903809, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0008222603937610984, + "policy/clipfrac_avg": 0.0158871840685606, + "policy/entropy_avg": 0.13180410861968994, + "step": 407, + "val/clipfrac_avg": 0.16988256573677063, + "val/num_eos_tokens": 286, + "val/ratio": 0.9913500547409058, + "val/ratio_var": 4.944593092659488e-05 + }, + { + "episode": 6528, + "epoch": 0.3296969696969697, + "eps": 0, + "loss/policy_avg": -0.465353399515152, + "loss/value_avg": 0.7926937341690063, + "lr": 8.022569983492781e-06, + "objective/entropy": -143.9917755126953, + "objective/kl": 22.473175048828125, + "objective/non_score_reward": -1.1236587762832642, + "objective/rlhf_reward": 10.821653366088867, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0006903298199176788, + "policy/clipfrac_avg": 0.02162603661417961, + "policy/entropy_avg": 0.16019591689109802, + "step": 408, + "val/clipfrac_avg": 0.14454638957977295, + "val/num_eos_tokens": 448, + "val/ratio": 0.9919313192367554, + "val/ratio_var": 4.090562288183719e-05 + }, + { + "episode": 6544, + "epoch": 0.3305050505050505, + "eps": 0, + "loss/policy_avg": 0.07690078765153885, + "loss/value_avg": 0.29455649852752686, + "lr": 8.011919008964779e-06, + "objective/entropy": -75.08297729492188, + "objective/kl": 22.801063537597656, + "objective/non_score_reward": -1.1400532722473145, + "objective/rlhf_reward": 9.375572204589844, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.0012238363269716501, + "policy/clipfrac_avg": 0.013967175967991352, + "policy/entropy_avg": 0.15617752075195312, + "step": 409, + "val/clipfrac_avg": 0.19421738386154175, + "val/num_eos_tokens": 56, + "val/ratio": 0.9893690347671509, + "val/ratio_var": 5.305988452164456e-05 + }, + { + "episode": 6560, + "epoch": 0.33131313131313134, + "eps": 0, + "loss/policy_avg": 0.10671384632587433, + "loss/value_avg": 0.24573181569576263, + "lr": 8.001246539897877e-06, + "objective/entropy": -171.51019287109375, + "objective/kl": 23.710628509521484, + "objective/non_score_reward": -1.1855316162109375, + "objective/rlhf_reward": 10.869155883789062, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.000929538335185498, + "policy/clipfrac_avg": 0.030162684619426727, + "policy/entropy_avg": 0.17933368682861328, + "step": 410, + "val/clipfrac_avg": 0.07078950107097626, + "val/num_eos_tokens": 479, + "val/ratio": 0.9917181730270386, + "val/ratio_var": 3.602935612434521e-05 + }, + { + "episode": 6576, + "epoch": 0.3321212121212121, + "eps": 0, + "loss/policy_avg": -0.28077971935272217, + "loss/value_avg": 1.3987998962402344, + "lr": 7.99055265245608e-06, + "objective/entropy": -92.83210754394531, + "objective/kl": 18.754131317138672, + "objective/non_score_reward": -0.937706470489502, + "objective/rlhf_reward": 11.253700256347656, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.00083785445895046, + "policy/clipfrac_avg": 0.008017129264771938, + "policy/entropy_avg": 0.15318012237548828, + "step": 411, + "val/clipfrac_avg": 0.18445537984371185, + "val/num_eos_tokens": 23, + "val/ratio": 0.9898405075073242, + "val/ratio_var": 7.417797314701602e-05 + }, + { + "episode": 6592, + "epoch": 0.3329292929292929, + "eps": 0, + "loss/policy_avg": -0.18175235390663147, + "loss/value_avg": 0.18787693977355957, + "lr": 7.979837422956236e-06, + "objective/entropy": -66.04624938964844, + "objective/kl": 25.964508056640625, + "objective/non_score_reward": -1.2982254028320312, + "objective/rlhf_reward": 9.779899597167969, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0010015174048021436, + "policy/clipfrac_avg": 0.00993548333644867, + "policy/entropy_avg": 0.17697501182556152, + "step": 412, + "val/clipfrac_avg": 0.04342540353536606, + "val/num_eos_tokens": 47, + "val/ratio": 0.9883245825767517, + "val/ratio_var": 5.8682868257164955e-05 + }, + { + "episode": 6608, + "epoch": 0.3337373737373737, + "eps": 0, + "loss/policy_avg": 0.22827568650245667, + "loss/value_avg": 0.5228079557418823, + "lr": 7.969100927867508e-06, + "objective/entropy": -34.08283233642578, + "objective/kl": 25.512779235839844, + "objective/non_score_reward": -1.2756389379501343, + "objective/rlhf_reward": 9.029048919677734, + "objective/scores": 10.3125, + "policy/approxkl_avg": 0.001080115558579564, + "policy/clipfrac_avg": 0.009306976571679115, + "policy/entropy_avg": 0.20532989501953125, + "step": 413, + "val/clipfrac_avg": 0.04794730246067047, + "val/num_eos_tokens": 18, + "val/ratio": 0.9874177575111389, + "val/ratio_var": 2.4738281354075298e-05 + }, + { + "episode": 6624, + "epoch": 0.33454545454545453, + "eps": 0, + "loss/policy_avg": -0.2480793595314026, + "loss/value_avg": 0.7881271243095398, + "lr": 7.958343243810818e-06, + "objective/entropy": -88.73225402832031, + "objective/kl": 17.2120304107666, + "objective/non_score_reward": -0.860601544380188, + "objective/rlhf_reward": 12.131586074829102, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0006878376007080078, + "policy/clipfrac_avg": 0.005827655084431171, + "policy/entropy_avg": 0.1268022060394287, + "step": 414, + "val/clipfrac_avg": 0.22013108432292938, + "val/num_eos_tokens": 113, + "val/ratio": 0.9918868541717529, + "val/ratio_var": 6.459503492806107e-05 + }, + { + "episode": 6640, + "epoch": 0.33535353535353535, + "eps": 0, + "loss/policy_avg": -0.007346153259277344, + "loss/value_avg": 1.3498705625534058, + "lr": 7.9475644475583e-06, + "objective/entropy": -65.54212951660156, + "objective/kl": 22.453895568847656, + "objective/non_score_reward": -1.1226948499679565, + "objective/rlhf_reward": 11.283555030822754, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008567589102312922, + "policy/clipfrac_avg": 0.005040471442043781, + "policy/entropy_avg": 0.16246891021728516, + "step": 415, + "val/clipfrac_avg": 0.10788097977638245, + "val/num_eos_tokens": 322, + "val/ratio": 0.9893018007278442, + "val/ratio_var": 4.8482721467735246e-05 + }, + { + "episode": 6656, + "epoch": 0.33616161616161616, + "eps": 0, + "loss/policy_avg": 0.31426429748535156, + "loss/value_avg": 2.21596097946167, + "lr": 7.93676461603276e-06, + "objective/entropy": -25.829357147216797, + "objective/kl": 13.174534797668457, + "objective/non_score_reward": -0.6587268114089966, + "objective/rlhf_reward": 10.313929557800293, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0008746474632062018, + "policy/clipfrac_avg": 0.006421862170100212, + "policy/entropy_avg": 0.19477272033691406, + "step": 416, + "val/clipfrac_avg": 0.15579935908317566, + "val/num_eos_tokens": 7, + "val/ratio": 0.9900082349777222, + "val/ratio_var": 9.775400394573808e-05 + }, + { + "episode": 6672, + "epoch": 0.336969696969697, + "eps": 0, + "loss/policy_avg": -0.5780159831047058, + "loss/value_avg": 0.7495456337928772, + "lr": 7.925943826307119e-06, + "objective/entropy": -116.59674072265625, + "objective/kl": 20.854877471923828, + "objective/non_score_reward": -1.0427439212799072, + "objective/rlhf_reward": 13.402568817138672, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000628868059720844, + "policy/clipfrac_avg": 0.003246631007641554, + "policy/entropy_avg": 0.12189757823944092, + "step": 417, + "val/clipfrac_avg": 0.10604649782180786, + "val/num_eos_tokens": 124, + "val/ratio": 0.9918810129165649, + "val/ratio_var": 3.4486944059608504e-05 + }, + { + "episode": 6688, + "epoch": 0.3377777777777778, + "eps": 0, + "loss/policy_avg": 0.12082649767398834, + "loss/value_avg": 0.7548036575317383, + "lr": 7.91510215560387e-06, + "objective/entropy": -129.1641082763672, + "objective/kl": 25.002452850341797, + "objective/non_score_reward": -1.2501227855682373, + "objective/rlhf_reward": 9.6483154296875, + "objective/scores": 10.875, + "policy/approxkl_avg": 0.0008685050415806472, + "policy/clipfrac_avg": 0.0092330202460289, + "policy/entropy_avg": 0.16512155532836914, + "step": 418, + "val/clipfrac_avg": 0.1482473611831665, + "val/num_eos_tokens": 431, + "val/ratio": 0.9887162446975708, + "val/ratio_var": 7.209738396340981e-05 + }, + { + "episode": 6704, + "epoch": 0.3385858585858586, + "eps": 0, + "loss/policy_avg": 0.3398967981338501, + "loss/value_avg": 1.4188909530639648, + "lr": 7.904239681294515e-06, + "objective/entropy": -99.79671478271484, + "objective/kl": 17.616220474243164, + "objective/non_score_reward": -0.8808110952377319, + "objective/rlhf_reward": 10.90825080871582, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0006818549009039998, + "policy/clipfrac_avg": 0.010476157069206238, + "policy/entropy_avg": 0.1477426290512085, + "step": 419, + "val/clipfrac_avg": 0.1959417462348938, + "val/num_eos_tokens": 87, + "val/ratio": 0.9921755790710449, + "val/ratio_var": 3.494379416224547e-05 + }, + { + "episode": 6720, + "epoch": 0.3393939393939394, + "eps": 0, + "loss/policy_avg": -0.20346836745738983, + "loss/value_avg": 1.3521277904510498, + "lr": 7.89335648089903e-06, + "objective/entropy": -143.52662658691406, + "objective/kl": 20.073959350585938, + "objective/non_score_reward": -1.0036978721618652, + "objective/rlhf_reward": 11.246301651000977, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007999450317583978, + "policy/clipfrac_avg": 0.005896029528230429, + "policy/entropy_avg": 0.1470661163330078, + "step": 420, + "val/clipfrac_avg": 0.10764417052268982, + "val/num_eos_tokens": 184, + "val/ratio": 0.9900174140930176, + "val/ratio_var": 5.0762690079864115e-05 + }, + { + "episode": 6736, + "epoch": 0.3402020202020202, + "eps": 0, + "loss/policy_avg": 0.3638865351676941, + "loss/value_avg": 0.498892605304718, + "lr": 7.882452632085296e-06, + "objective/entropy": -125.03632354736328, + "objective/kl": 24.23324203491211, + "objective/non_score_reward": -1.2116621732711792, + "objective/rlhf_reward": 9.202400207519531, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0008304300718009472, + "policy/clipfrac_avg": 0.004866223782300949, + "policy/entropy_avg": 0.15968847274780273, + "step": 421, + "val/clipfrac_avg": 0.11759978532791138, + "val/num_eos_tokens": 266, + "val/ratio": 0.9900087118148804, + "val/ratio_var": 5.0026934331981465e-05 + }, + { + "episode": 6752, + "epoch": 0.341010101010101, + "eps": 0, + "loss/policy_avg": 0.18302330374717712, + "loss/value_avg": 0.675520122051239, + "lr": 7.871528212668556e-06, + "objective/entropy": -25.790729522705078, + "objective/kl": 24.247493743896484, + "objective/non_score_reward": -1.2123748064041138, + "objective/rlhf_reward": 10.975125312805176, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0010115428594872355, + "policy/clipfrac_avg": 0.006701755337417126, + "policy/entropy_avg": 0.16680622100830078, + "step": 422, + "val/clipfrac_avg": 0.14078916609287262, + "val/num_eos_tokens": 30, + "val/ratio": 0.988139271736145, + "val/ratio_var": 6.585121445823461e-05 + }, + { + "episode": 6768, + "epoch": 0.3418181818181818, + "eps": 0, + "loss/policy_avg": 0.1767496019601822, + "loss/value_avg": 0.8422822952270508, + "lr": 7.860583300610849e-06, + "objective/entropy": -99.92378234863281, + "objective/kl": 23.447364807128906, + "objective/non_score_reward": -1.1723682880401611, + "objective/rlhf_reward": 10.062005996704102, + "objective/scores": 11.25, + "policy/approxkl_avg": 0.0008515919907949865, + "policy/clipfrac_avg": 0.011789527721703053, + "policy/entropy_avg": 0.1661376953125, + "step": 423, + "val/clipfrac_avg": 0.07063312828540802, + "val/num_eos_tokens": 82, + "val/ratio": 0.9900652766227722, + "val/ratio_var": 2.7732188755180687e-05 + }, + { + "episode": 6784, + "epoch": 0.3426262626262626, + "eps": 0, + "loss/policy_avg": -0.09848126024007797, + "loss/value_avg": 0.2839937210083008, + "lr": 7.849617974020467e-06, + "objective/entropy": -87.34598541259766, + "objective/kl": 22.923152923583984, + "objective/non_score_reward": -1.1461577415466309, + "objective/rlhf_reward": 10.056966781616211, + "objective/scores": 11.1875, + "policy/approxkl_avg": 0.001004643039777875, + "policy/clipfrac_avg": 0.01210677158087492, + "policy/entropy_avg": 0.15830230712890625, + "step": 424, + "val/clipfrac_avg": 0.06793520599603653, + "val/num_eos_tokens": 316, + "val/ratio": 0.9890733361244202, + "val/ratio_var": 4.017764877062291e-05 + }, + { + "episode": 6800, + "epoch": 0.3434343434343434, + "eps": 0, + "loss/policy_avg": -0.4337696433067322, + "loss/value_avg": 1.939749002456665, + "lr": 7.838632311151384e-06, + "objective/entropy": -24.3656005859375, + "objective/kl": 20.979629516601562, + "objective/non_score_reward": -1.0489814281463623, + "objective/rlhf_reward": 10.318206787109375, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.0009983977070078254, + "policy/clipfrac_avg": 0.006080445367842913, + "policy/entropy_avg": 0.1742689609527588, + "step": 425, + "val/clipfrac_avg": 0.10390697419643402, + "val/num_eos_tokens": 217, + "val/ratio": 0.9901939630508423, + "val/ratio_var": 5.642051110044122e-05 + }, + { + "episode": 6816, + "epoch": 0.34424242424242424, + "eps": 0, + "loss/policy_avg": 0.0010461192578077316, + "loss/value_avg": 0.22941997647285461, + "lr": 7.827626390402707e-06, + "objective/entropy": -38.45948791503906, + "objective/kl": 27.880338668823242, + "objective/non_score_reward": -1.3940168619155884, + "objective/rlhf_reward": 9.11379623413086, + "objective/scores": 10.5, + "policy/approxkl_avg": 0.00097821862436831, + "policy/clipfrac_avg": 0.006474747322499752, + "policy/entropy_avg": 0.18066978454589844, + "step": 426, + "val/clipfrac_avg": 0.03179371729493141, + "val/num_eos_tokens": 113, + "val/ratio": 0.9881677627563477, + "val/ratio_var": 3.830140849458985e-05 + }, + { + "episode": 6832, + "epoch": 0.34505050505050505, + "eps": 0, + "loss/policy_avg": -0.07846693694591522, + "loss/value_avg": 0.26400119066238403, + "lr": 7.81660029031811e-06, + "objective/entropy": -61.58184051513672, + "objective/kl": 30.09839630126953, + "objective/non_score_reward": -1.5049197673797607, + "objective/rlhf_reward": 9.81539249420166, + "objective/scores": 11.3125, + "policy/approxkl_avg": 0.0010863475035876036, + "policy/clipfrac_avg": 0.01059107854962349, + "policy/entropy_avg": 0.15493392944335938, + "step": 427, + "val/clipfrac_avg": 0.037561144679784775, + "val/num_eos_tokens": 138, + "val/ratio": 0.9897431135177612, + "val/ratio_var": 4.0529008401790634e-05 + }, + { + "episode": 6848, + "epoch": 0.34585858585858587, + "eps": 0, + "loss/policy_avg": -0.38922715187072754, + "loss/value_avg": 0.6340113282203674, + "lr": 7.805554089585284e-06, + "objective/entropy": -48.85191345214844, + "objective/kl": 21.427749633789062, + "objective/non_score_reward": -1.0713874101638794, + "objective/rlhf_reward": 9.936426162719727, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0023591392673552036, + "policy/clipfrac_avg": 0.019066959619522095, + "policy/entropy_avg": 0.20233774185180664, + "step": 428, + "val/clipfrac_avg": 0.08839962631464005, + "val/num_eos_tokens": 56, + "val/ratio": 0.9836812019348145, + "val/ratio_var": 0.0002559312852099538 + }, + { + "episode": 6864, + "epoch": 0.3466666666666667, + "eps": 0, + "loss/policy_avg": -0.4161893427371979, + "loss/value_avg": 1.773421049118042, + "lr": 7.794487867035358e-06, + "objective/entropy": -94.49333190917969, + "objective/kl": 19.139610290527344, + "objective/non_score_reward": -0.9569804668426514, + "objective/rlhf_reward": 13.48051929473877, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010149467270821333, + "policy/clipfrac_avg": 0.009838546626269817, + "policy/entropy_avg": 0.16881978511810303, + "step": 429, + "val/clipfrac_avg": 0.18258625268936157, + "val/num_eos_tokens": 84, + "val/ratio": 0.9892622232437134, + "val/ratio_var": 8.773741137702018e-05 + }, + { + "episode": 6880, + "epoch": 0.3474747474747475, + "eps": 0, + "loss/policy_avg": -0.36678922176361084, + "loss/value_avg": 0.3908446133136749, + "lr": 7.783401701642356e-06, + "objective/entropy": -144.0445098876953, + "objective/kl": 28.28658676147461, + "objective/non_score_reward": -1.4143294095993042, + "objective/rlhf_reward": 11.101295471191406, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.0007755355327390134, + "policy/clipfrac_avg": 0.005785915069282055, + "policy/entropy_avg": 0.15067648887634277, + "step": 430, + "val/clipfrac_avg": 0.11318591237068176, + "val/num_eos_tokens": 683, + "val/ratio": 0.9902333617210388, + "val/ratio_var": 4.714354145107791e-05 + }, + { + "episode": 6896, + "epoch": 0.3482828282828283, + "eps": 0, + "loss/policy_avg": -0.07360352575778961, + "loss/value_avg": 0.4391564130783081, + "lr": 7.772295672522615e-06, + "objective/entropy": -88.11723327636719, + "objective/kl": 24.70610809326172, + "objective/non_score_reward": -1.2353053092956543, + "objective/rlhf_reward": 11.608444213867188, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0012505800696089864, + "policy/clipfrac_avg": 0.017380211502313614, + "policy/entropy_avg": 0.1649489402770996, + "step": 431, + "val/clipfrac_avg": 0.08813799917697906, + "val/num_eos_tokens": 396, + "val/ratio": 0.9886548519134521, + "val/ratio_var": 8.897272346075624e-05 + }, + { + "episode": 6912, + "epoch": 0.3490909090909091, + "eps": 0, + "loss/policy_avg": -0.3927380442619324, + "loss/value_avg": 0.7407118082046509, + "lr": 7.761169858934238e-06, + "objective/entropy": -98.53260803222656, + "objective/kl": 25.28207015991211, + "objective/non_score_reward": -1.2641034126281738, + "objective/rlhf_reward": 10.774959564208984, + "objective/scores": 12.0625, + "policy/approxkl_avg": 0.0009465490002185106, + "policy/clipfrac_avg": 0.017566177994012833, + "policy/entropy_avg": 0.17182016372680664, + "step": 432, + "val/clipfrac_avg": 0.05415056645870209, + "val/num_eos_tokens": 205, + "val/ratio": 0.9887499809265137, + "val/ratio_var": 5.970060738036409e-05 + }, + { + "episode": 6928, + "epoch": 0.3498989898989899, + "eps": 0, + "loss/policy_avg": 0.07162998616695404, + "loss/value_avg": 0.6784456968307495, + "lr": 7.750024340276512e-06, + "objective/entropy": -105.76652526855469, + "objective/kl": 29.98111915588379, + "objective/non_score_reward": -1.4990559816360474, + "objective/rlhf_reward": 11.250944137573242, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0008196796989068389, + "policy/clipfrac_avg": 0.014055829495191574, + "policy/entropy_avg": 0.16859722137451172, + "step": 433, + "val/clipfrac_avg": 0.10481823980808258, + "val/num_eos_tokens": 508, + "val/ratio": 0.9896249771118164, + "val/ratio_var": 6.0306902014417574e-05 + }, + { + "episode": 6944, + "epoch": 0.3507070707070707, + "eps": 0, + "loss/policy_avg": 0.21741759777069092, + "loss/value_avg": 0.4865216314792633, + "lr": 7.738859196089358e-06, + "objective/entropy": -118.70050048828125, + "objective/kl": 24.68207550048828, + "objective/non_score_reward": -1.2341036796569824, + "objective/rlhf_reward": 10.75808334350586, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000771071354392916, + "policy/clipfrac_avg": 0.012887731194496155, + "policy/entropy_avg": 0.12623023986816406, + "step": 434, + "val/clipfrac_avg": 0.1991758644580841, + "val/num_eos_tokens": 560, + "val/ratio": 0.9922160506248474, + "val/ratio_var": 3.5436314647085965e-05 + }, + { + "episode": 6960, + "epoch": 0.3515151515151515, + "eps": 0, + "loss/policy_avg": -0.07316741347312927, + "loss/value_avg": 0.22129662334918976, + "lr": 7.727674506052744e-06, + "objective/entropy": -127.72758483886719, + "objective/kl": 31.92902374267578, + "objective/non_score_reward": -1.5964511632919312, + "objective/rlhf_reward": 10.106673240661621, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.0008912943303585052, + "policy/clipfrac_avg": 0.006930951029062271, + "policy/entropy_avg": 0.14632463455200195, + "step": 435, + "val/clipfrac_avg": 0.07383783161640167, + "val/num_eos_tokens": 833, + "val/ratio": 0.989611029624939, + "val/ratio_var": 6.377204408636317e-05 + }, + { + "episode": 6976, + "epoch": 0.3523232323232323, + "eps": 0, + "loss/policy_avg": 0.27702921628952026, + "loss/value_avg": 0.9074195623397827, + "lr": 7.716470349986134e-06, + "objective/entropy": -99.71429443359375, + "objective/kl": 32.4725227355957, + "objective/non_score_reward": -1.6236259937286377, + "objective/rlhf_reward": 10.235749244689941, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0009519133600406349, + "policy/clipfrac_avg": 0.015381633304059505, + "policy/entropy_avg": 0.17629116773605347, + "step": 436, + "val/clipfrac_avg": 0.3032349646091461, + "val/num_eos_tokens": 165, + "val/ratio": 0.9885252714157104, + "val/ratio_var": 7.608420855831355e-05 + }, + { + "episode": 6992, + "epoch": 0.35313131313131313, + "eps": 0, + "loss/policy_avg": -0.10949157923460007, + "loss/value_avg": 0.7171697020530701, + "lr": 7.70524680784791e-06, + "objective/entropy": -67.62409973144531, + "objective/kl": 29.33208656311035, + "objective/non_score_reward": -1.466604471206665, + "objective/rlhf_reward": 12.588083267211914, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009833010844886303, + "policy/clipfrac_avg": 0.006308242678642273, + "policy/entropy_avg": 0.14640045166015625, + "step": 437, + "val/clipfrac_avg": 0.22335688769817352, + "val/num_eos_tokens": 42, + "val/ratio": 0.9893021583557129, + "val/ratio_var": 3.676968117360957e-05 + }, + { + "episode": 7008, + "epoch": 0.35393939393939394, + "eps": 0, + "loss/policy_avg": 0.20945397019386292, + "loss/value_avg": 0.2541009783744812, + "lr": 7.694003959734802e-06, + "objective/entropy": -34.55720520019531, + "objective/kl": 29.602069854736328, + "objective/non_score_reward": -1.4801034927368164, + "objective/rlhf_reward": 10.379271507263184, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.001006154459901154, + "policy/clipfrac_avg": 0.005651135928928852, + "policy/entropy_avg": 0.17934393882751465, + "step": 438, + "val/clipfrac_avg": 0.1426633596420288, + "val/num_eos_tokens": 142, + "val/ratio": 0.9886967539787292, + "val/ratio_var": 4.757837450597435e-05 + }, + { + "episode": 7024, + "epoch": 0.35474747474747476, + "eps": 0, + "loss/policy_avg": 0.04139135777950287, + "loss/value_avg": 0.8019120097160339, + "lr": 7.682741885881314e-06, + "objective/entropy": -95.60530090332031, + "objective/kl": 25.049076080322266, + "objective/non_score_reward": -1.2524539232254028, + "objective/rlhf_reward": 10.74754524230957, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0030104375910013914, + "policy/clipfrac_avg": 0.009499769657850266, + "policy/entropy_avg": 0.14387786388397217, + "step": 439, + "val/clipfrac_avg": 0.11603914201259613, + "val/num_eos_tokens": 222, + "val/ratio": 0.9905297160148621, + "val/ratio_var": 5.1482784328982234e-05 + }, + { + "episode": 7040, + "epoch": 0.35555555555555557, + "eps": 0, + "loss/policy_avg": 0.2531706392765045, + "loss/value_avg": 0.6514580845832825, + "lr": 7.671460666659161e-06, + "objective/entropy": -84.95793151855469, + "objective/kl": 28.195911407470703, + "objective/non_score_reward": -1.4097956418991089, + "objective/rlhf_reward": 10.613641738891602, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0007476760656572878, + "policy/clipfrac_avg": 0.006465469021350145, + "policy/entropy_avg": 0.12859344482421875, + "step": 440, + "val/clipfrac_avg": 0.1621960699558258, + "val/num_eos_tokens": 104, + "val/ratio": 0.9909597635269165, + "val/ratio_var": 5.390801379689947e-05 + }, + { + "episode": 7056, + "epoch": 0.3563636363636364, + "eps": 0, + "loss/policy_avg": -0.04607268422842026, + "loss/value_avg": 0.24199238419532776, + "lr": 7.660160382576683e-06, + "objective/entropy": -88.81119537353516, + "objective/kl": 38.44916915893555, + "objective/non_score_reward": -1.9224586486816406, + "objective/rlhf_reward": 10.08535385131836, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.000971765024587512, + "policy/clipfrac_avg": 0.007417440880089998, + "policy/entropy_avg": 0.18393898010253906, + "step": 441, + "val/clipfrac_avg": 0.10574978590011597, + "val/num_eos_tokens": 173, + "val/ratio": 0.9876141548156738, + "val/ratio_var": 4.917777914670296e-05 + }, + { + "episode": 7072, + "epoch": 0.3571717171717172, + "eps": 0, + "loss/policy_avg": -0.10014820098876953, + "loss/value_avg": 0.7047788500785828, + "lr": 7.648841114278278e-06, + "objective/entropy": -92.62014770507812, + "objective/kl": 29.281410217285156, + "objective/non_score_reward": -1.4640705585479736, + "objective/rlhf_reward": 11.551554679870605, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0011433872859925032, + "policy/clipfrac_avg": 0.012358217500150204, + "policy/entropy_avg": 0.16633868217468262, + "step": 442, + "val/clipfrac_avg": 0.07325705885887146, + "val/num_eos_tokens": 292, + "val/ratio": 0.9898238778114319, + "val/ratio_var": 5.3111554734641686e-05 + }, + { + "episode": 7088, + "epoch": 0.35797979797979795, + "eps": 0, + "loss/policy_avg": 0.2637108862400055, + "loss/value_avg": 0.8885920643806458, + "lr": 7.637502942543825e-06, + "objective/entropy": -49.81909942626953, + "objective/kl": 30.107555389404297, + "objective/non_score_reward": -1.5053777694702148, + "objective/rlhf_reward": 9.268058776855469, + "objective/scores": 10.75, + "policy/approxkl_avg": 0.0012591599952429533, + "policy/clipfrac_avg": 0.015652721747756004, + "policy/entropy_avg": 0.16948866844177246, + "step": 443, + "val/clipfrac_avg": 0.06298521906137466, + "val/num_eos_tokens": 108, + "val/ratio": 0.988282322883606, + "val/ratio_var": 3.1702376872999594e-05 + }, + { + "episode": 7104, + "epoch": 0.35878787878787877, + "eps": 0, + "loss/policy_avg": 0.2935371994972229, + "loss/value_avg": 0.7965297698974609, + "lr": 7.626145948288107e-06, + "objective/entropy": -128.95480346679688, + "objective/kl": 29.099103927612305, + "objective/non_score_reward": -1.4549552202224731, + "objective/rlhf_reward": 10.170044898986816, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0009231308940798044, + "policy/clipfrac_avg": 0.007753930985927582, + "policy/entropy_avg": 0.17519092559814453, + "step": 444, + "val/clipfrac_avg": 0.12779459357261658, + "val/num_eos_tokens": 712, + "val/ratio": 0.9897068738937378, + "val/ratio_var": 4.8030484322225675e-05 + }, + { + "episode": 7120, + "epoch": 0.3595959595959596, + "eps": 0, + "loss/policy_avg": -0.10212291777133942, + "loss/value_avg": 0.41369742155075073, + "lr": 7.614770212560234e-06, + "objective/entropy": -141.06298828125, + "objective/kl": 43.964115142822266, + "objective/non_score_reward": -2.1982059478759766, + "objective/rlhf_reward": 8.778356552124023, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0010534923058003187, + "policy/clipfrac_avg": 0.00662950798869133, + "policy/entropy_avg": 0.16738319396972656, + "step": 445, + "val/clipfrac_avg": 0.20534712076187134, + "val/num_eos_tokens": 35, + "val/ratio": 0.9893473386764526, + "val/ratio_var": 6.579797627637163e-05 + }, + { + "episode": 7136, + "epoch": 0.3604040404040404, + "eps": 0, + "loss/policy_avg": 0.043000537902116776, + "loss/value_avg": 0.2879178524017334, + "lr": 7.603375816543065e-06, + "objective/entropy": -137.25640869140625, + "objective/kl": 40.433448791503906, + "objective/non_score_reward": -2.0216727256774902, + "objective/rlhf_reward": 9.408015251159668, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.0011193909449502826, + "policy/clipfrac_avg": 0.009532807394862175, + "policy/entropy_avg": 0.1789846420288086, + "step": 446, + "val/clipfrac_avg": 0.15842124819755554, + "val/num_eos_tokens": 429, + "val/ratio": 0.9881588816642761, + "val/ratio_var": 8.264256757684052e-05 + }, + { + "episode": 7152, + "epoch": 0.3612121212121212, + "eps": 0, + "loss/policy_avg": -0.09768345952033997, + "loss/value_avg": 0.20441338419914246, + "lr": 7.591962841552627e-06, + "objective/entropy": -107.86872863769531, + "objective/kl": 39.474517822265625, + "objective/non_score_reward": -1.9737260341644287, + "objective/rlhf_reward": 9.784086227416992, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.000841513741761446, + "policy/clipfrac_avg": 0.006178049370646477, + "policy/entropy_avg": 0.1564655303955078, + "step": 447, + "val/clipfrac_avg": 0.12125139683485031, + "val/num_eos_tokens": 35, + "val/ratio": 0.9900634288787842, + "val/ratio_var": 4.8899408284341916e-05 + }, + { + "episode": 7168, + "epoch": 0.362020202020202, + "eps": 0, + "loss/policy_avg": -0.23434095084667206, + "loss/value_avg": 0.9878807663917542, + "lr": 7.580531369037534e-06, + "objective/entropy": -65.5897216796875, + "objective/kl": 34.39695739746094, + "objective/non_score_reward": -1.7198479175567627, + "objective/rlhf_reward": 10.537964820861816, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009984057396650314, + "policy/clipfrac_avg": 0.005852546542882919, + "policy/entropy_avg": 0.17050552368164062, + "step": 448, + "val/clipfrac_avg": 0.03700433671474457, + "val/num_eos_tokens": 266, + "val/ratio": 0.9879032373428345, + "val/ratio_var": 6.289610610110685e-05 + }, + { + "episode": 7184, + "epoch": 0.36282828282828283, + "eps": 0, + "loss/policy_avg": -0.09545287489891052, + "loss/value_avg": 0.35513487458229065, + "lr": 7.569081480578413e-06, + "objective/entropy": -42.06928634643555, + "objective/kl": 30.470748901367188, + "objective/non_score_reward": -1.5235373973846436, + "objective/rlhf_reward": 10.898337364196777, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0008559878915548325, + "policy/clipfrac_avg": 0.008136066608130932, + "policy/entropy_avg": 0.14156532287597656, + "step": 449, + "val/clipfrac_avg": 0.13745607435703278, + "val/num_eos_tokens": 263, + "val/ratio": 0.9917177557945251, + "val/ratio_var": 5.956144741503522e-05 + }, + { + "episode": 7200, + "epoch": 0.36363636363636365, + "eps": 0, + "loss/policy_avg": -0.09895340353250504, + "loss/value_avg": 0.6456364989280701, + "lr": 7.55761325788731e-06, + "objective/entropy": -166.6522674560547, + "objective/kl": 36.183135986328125, + "objective/non_score_reward": -1.8091567754745483, + "objective/rlhf_reward": 9.18303108215332, + "objective/scores": 11.0, + "policy/approxkl_avg": 0.0012210204731673002, + "policy/clipfrac_avg": 0.011477088555693626, + "policy/entropy_avg": 0.15171527862548828, + "step": 450, + "val/clipfrac_avg": 0.14100708067417145, + "val/num_eos_tokens": 196, + "val/ratio": 0.9893430471420288, + "val/ratio_var": 6.879774446133524e-05 + }, + { + "episode": 7216, + "epoch": 0.36444444444444446, + "eps": 0, + "loss/policy_avg": -0.034445568919181824, + "loss/value_avg": 1.0610827207565308, + "lr": 7.546126782807117e-06, + "objective/entropy": -95.6431884765625, + "objective/kl": 28.169822692871094, + "objective/non_score_reward": -1.4084912538528442, + "objective/rlhf_reward": 10.372758865356445, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009328583255410194, + "policy/clipfrac_avg": 0.008236020803451538, + "policy/entropy_avg": 0.1556539535522461, + "step": 451, + "val/clipfrac_avg": 0.06177588552236557, + "val/num_eos_tokens": 303, + "val/ratio": 0.9893108010292053, + "val/ratio_var": 4.2810836021089926e-05 + }, + { + "episode": 7232, + "epoch": 0.3652525252525253, + "eps": 0, + "loss/policy_avg": 0.26358139514923096, + "loss/value_avg": 0.5271927118301392, + "lr": 7.534622137310983e-06, + "objective/entropy": -65.11871337890625, + "objective/kl": 26.70556640625, + "objective/non_score_reward": -1.3352782726287842, + "objective/rlhf_reward": 10.086596488952637, + "objective/scores": 11.4375, + "policy/approxkl_avg": 0.001023412449285388, + "policy/clipfrac_avg": 0.013055086135864258, + "policy/entropy_avg": 0.1628122329711914, + "step": 452, + "val/clipfrac_avg": 0.24446935951709747, + "val/num_eos_tokens": 248, + "val/ratio": 0.9888921976089478, + "val/ratio_var": 7.8632976510562e-05 + }, + { + "episode": 7248, + "epoch": 0.3660606060606061, + "eps": 0, + "loss/policy_avg": -0.20332151651382446, + "loss/value_avg": 2.0998940467834473, + "lr": 7.52309940350173e-06, + "objective/entropy": -100.48619079589844, + "objective/kl": 23.723403930664062, + "objective/non_score_reward": -1.186170220375061, + "objective/rlhf_reward": 12.63414192199707, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0010372435208410025, + "policy/clipfrac_avg": 0.009655164554715157, + "policy/entropy_avg": 0.16978693008422852, + "step": 453, + "val/clipfrac_avg": 0.28355222940444946, + "val/num_eos_tokens": 168, + "val/ratio": 0.9885410070419312, + "val/ratio_var": 5.881414836039767e-05 + }, + { + "episode": 7264, + "epoch": 0.36686868686868684, + "eps": 0, + "loss/policy_avg": -0.20516440272331238, + "loss/value_avg": 0.5702744722366333, + "lr": 7.511558663611265e-06, + "objective/entropy": -121.33277130126953, + "objective/kl": 27.353843688964844, + "objective/non_score_reward": -1.367692232131958, + "objective/rlhf_reward": 12.569808006286621, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009084678022190928, + "policy/clipfrac_avg": 0.010569443926215172, + "policy/entropy_avg": 0.1651425063610077, + "step": 454, + "val/clipfrac_avg": 0.10168075561523438, + "val/num_eos_tokens": 240, + "val/ratio": 0.9904431104660034, + "val/ratio_var": 4.341174280853011e-05 + }, + { + "episode": 7280, + "epoch": 0.36767676767676766, + "eps": 0, + "loss/policy_avg": -0.11988245695829391, + "loss/value_avg": 0.24273253977298737, + "lr": 7.500000000000001e-06, + "objective/entropy": -94.4637222290039, + "objective/kl": 27.129352569580078, + "objective/non_score_reward": -1.356467604637146, + "objective/rlhf_reward": 10.256814002990723, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0010198624804615974, + "policy/clipfrac_avg": 0.00805464293807745, + "policy/entropy_avg": 0.1758131980895996, + "step": 455, + "val/clipfrac_avg": 0.05403827503323555, + "val/num_eos_tokens": 216, + "val/ratio": 0.9884688854217529, + "val/ratio_var": 6.81005185469985e-05 + }, + { + "episode": 7296, + "epoch": 0.36848484848484847, + "eps": 0, + "loss/policy_avg": 0.15036238729953766, + "loss/value_avg": 1.3548328876495361, + "lr": 7.488423495156258e-06, + "objective/entropy": -71.29700469970703, + "objective/kl": 30.76184844970703, + "objective/non_score_reward": -1.5380924940109253, + "objective/rlhf_reward": 11.415032386779785, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0010650574695318937, + "policy/clipfrac_avg": 0.013442248106002808, + "policy/entropy_avg": 0.1958608627319336, + "step": 456, + "val/clipfrac_avg": 0.18505151569843292, + "val/num_eos_tokens": 102, + "val/ratio": 0.9862017631530762, + "val/ratio_var": 8.930258627515286e-05 + }, + { + "episode": 7312, + "epoch": 0.3692929292929293, + "eps": 0, + "loss/policy_avg": -0.03644943982362747, + "loss/value_avg": 0.6813304424285889, + "lr": 7.47682923169568e-06, + "objective/entropy": -110.99166870117188, + "objective/kl": 24.803741455078125, + "objective/non_score_reward": -1.240187168121338, + "objective/rlhf_reward": 12.86137580871582, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009097999427467585, + "policy/clipfrac_avg": 0.014383969828486443, + "policy/entropy_avg": 0.14606904983520508, + "step": 457, + "val/clipfrac_avg": 0.1426132172346115, + "val/num_eos_tokens": 396, + "val/ratio": 0.9902492165565491, + "val/ratio_var": 5.0029055273626e-05 + }, + { + "episode": 7328, + "epoch": 0.3701010101010101, + "eps": 0, + "loss/policy_avg": -0.08388765901327133, + "loss/value_avg": 0.9905740022659302, + "lr": 7.4652172923606494e-06, + "objective/entropy": -49.973045349121094, + "objective/kl": 22.956039428710938, + "objective/non_score_reward": -1.1478021144866943, + "objective/rlhf_reward": 10.875635147094727, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010175809729844332, + "policy/clipfrac_avg": 0.011250700801610947, + "policy/entropy_avg": 0.20994114875793457, + "step": 458, + "val/clipfrac_avg": 0.21617577970027924, + "val/num_eos_tokens": 59, + "val/ratio": 0.9885356426239014, + "val/ratio_var": 7.953841850394383e-05 + }, + { + "episode": 7344, + "epoch": 0.3709090909090909, + "eps": 0, + "loss/policy_avg": 0.12381671369075775, + "loss/value_avg": 0.41641736030578613, + "lr": 7.453587760019691e-06, + "objective/entropy": -133.6717529296875, + "objective/kl": 27.69005012512207, + "objective/non_score_reward": -1.3845025300979614, + "objective/rlhf_reward": 10.365497589111328, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0008332305587828159, + "policy/clipfrac_avg": 0.01680421084165573, + "policy/entropy_avg": 0.13964557647705078, + "step": 459, + "val/clipfrac_avg": 0.23695185780525208, + "val/num_eos_tokens": 526, + "val/ratio": 0.9924179315567017, + "val/ratio_var": 7.448666292475536e-05 + }, + { + "episode": 7360, + "epoch": 0.3717171717171717, + "eps": 0, + "loss/policy_avg": -0.17677900195121765, + "loss/value_avg": 2.0295965671539307, + "lr": 7.441940717666877e-06, + "objective/entropy": -37.12174987792969, + "objective/kl": 25.1827392578125, + "objective/non_score_reward": -1.2591369152069092, + "objective/rlhf_reward": 12.850236892700195, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0010175569914281368, + "policy/clipfrac_avg": 0.010180214419960976, + "policy/entropy_avg": 0.1750507354736328, + "step": 460, + "val/clipfrac_avg": 0.2530834674835205, + "val/num_eos_tokens": 219, + "val/ratio": 0.9883590936660767, + "val/ratio_var": 5.4351363360183313e-05 + }, + { + "episode": 7376, + "epoch": 0.37252525252525254, + "eps": 0, + "loss/policy_avg": 0.3645497262477875, + "loss/value_avg": 0.820085346698761, + "lr": 7.4302762484212465e-06, + "objective/entropy": -70.19863891601562, + "objective/kl": 26.147159576416016, + "objective/non_score_reward": -1.307357907295227, + "objective/rlhf_reward": 11.684829711914062, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0010134675540030003, + "policy/clipfrac_avg": 0.018840627744793892, + "policy/entropy_avg": 0.17819499969482422, + "step": 461, + "val/clipfrac_avg": 0.1551748812198639, + "val/num_eos_tokens": 67, + "val/ratio": 0.9893249869346619, + "val/ratio_var": 4.883955989498645e-05 + }, + { + "episode": 7392, + "epoch": 0.37333333333333335, + "eps": 0, + "loss/policy_avg": 0.3121355473995209, + "loss/value_avg": 0.5492252111434937, + "lr": 7.4185944355261996e-06, + "objective/entropy": -105.91459655761719, + "objective/kl": 23.205120086669922, + "objective/non_score_reward": -1.160256028175354, + "objective/rlhf_reward": 11.042869567871094, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0008878573426045477, + "policy/clipfrac_avg": 0.0075564016588032246, + "policy/entropy_avg": 0.16464996337890625, + "step": 462, + "val/clipfrac_avg": 0.09090471267700195, + "val/num_eos_tokens": 19, + "val/ratio": 0.9889609813690186, + "val/ratio_var": 4.3533022108022124e-05 + }, + { + "episode": 7408, + "epoch": 0.37414141414141416, + "eps": 0, + "loss/policy_avg": -0.22459957003593445, + "loss/value_avg": 0.6871347427368164, + "lr": 7.406895362348916e-06, + "objective/entropy": -67.8624267578125, + "objective/kl": 24.100055694580078, + "objective/non_score_reward": -1.2050029039382935, + "objective/rlhf_reward": 12.65437126159668, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0009261409286409616, + "policy/clipfrac_avg": 0.01092602964490652, + "policy/entropy_avg": 0.17773854732513428, + "step": 463, + "val/clipfrac_avg": 0.12751662731170654, + "val/num_eos_tokens": 290, + "val/ratio": 0.9884368181228638, + "val/ratio_var": 7.627475133631378e-05 + }, + { + "episode": 7424, + "epoch": 0.374949494949495, + "eps": 0, + "loss/policy_avg": 0.10064420849084854, + "loss/value_avg": 0.42092299461364746, + "lr": 7.395179112379746e-06, + "objective/entropy": -128.5548095703125, + "objective/kl": 25.083354949951172, + "objective/non_score_reward": -1.2541677951812744, + "objective/rlhf_reward": 10.964582443237305, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0009050636435858905, + "policy/clipfrac_avg": 0.011509673669934273, + "policy/entropy_avg": 0.1595158576965332, + "step": 464, + "val/clipfrac_avg": 0.08698742836713791, + "val/num_eos_tokens": 233, + "val/ratio": 0.9895754456520081, + "val/ratio_var": 0.00011247485235799104 + }, + { + "episode": 7440, + "epoch": 0.37575757575757573, + "eps": 0, + "loss/policy_avg": 0.18652041256427765, + "loss/value_avg": 0.6958376169204712, + "lr": 7.383445769231628e-06, + "objective/entropy": -212.094482421875, + "objective/kl": 23.573081970214844, + "objective/non_score_reward": -1.1786541938781738, + "objective/rlhf_reward": 11.715876579284668, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0007067503756843507, + "policy/clipfrac_avg": 0.011690440587699413, + "policy/entropy_avg": 0.16191935539245605, + "step": 465, + "val/clipfrac_avg": 0.27865731716156006, + "val/num_eos_tokens": 2015, + "val/ratio": 0.991563081741333, + "val/ratio_var": 4.854411963606253e-05 + }, + { + "episode": 7456, + "epoch": 0.37656565656565655, + "eps": 0, + "loss/policy_avg": -0.026313835754990578, + "loss/value_avg": 0.3103347420692444, + "lr": 7.371695416639481e-06, + "objective/entropy": -52.99230194091797, + "objective/kl": 32.347415924072266, + "objective/non_score_reward": -1.6173707246780396, + "objective/rlhf_reward": 11.49981689453125, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010288862977176905, + "policy/clipfrac_avg": 0.004554249346256256, + "policy/entropy_avg": 0.2293071746826172, + "step": 466, + "val/clipfrac_avg": 0.03159730136394501, + "val/num_eos_tokens": 52, + "val/ratio": 0.9858705997467041, + "val/ratio_var": 7.237601676024497e-05 + }, + { + "episode": 7472, + "epoch": 0.37737373737373736, + "eps": 0, + "loss/policy_avg": -0.1737292855978012, + "loss/value_avg": 0.4936577081680298, + "lr": 7.359928138459615e-06, + "objective/entropy": -167.1389617919922, + "objective/kl": 24.184755325317383, + "objective/non_score_reward": -1.2092376947402954, + "objective/rlhf_reward": 11.716543197631836, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.0008187098428606987, + "policy/clipfrac_avg": 0.012958062812685966, + "policy/entropy_avg": 0.2136235237121582, + "step": 467, + "val/clipfrac_avg": 0.12412194907665253, + "val/num_eos_tokens": 789, + "val/ratio": 0.9899157881736755, + "val/ratio_var": 5.196731217438355e-05 + }, + { + "episode": 7488, + "epoch": 0.3781818181818182, + "eps": 0, + "loss/policy_avg": -0.2519969046115875, + "loss/value_avg": 0.6479045152664185, + "lr": 7.348144018669129e-06, + "objective/entropy": -99.32038879394531, + "objective/kl": 26.607398986816406, + "objective/non_score_reward": -1.3303699493408203, + "objective/rlhf_reward": 10.39619255065918, + "objective/scores": 11.75, + "policy/approxkl_avg": 0.0009252649615518749, + "policy/clipfrac_avg": 0.005941381677985191, + "policy/entropy_avg": 0.18393898010253906, + "step": 468, + "val/clipfrac_avg": 0.08323413878679276, + "val/num_eos_tokens": 578, + "val/ratio": 0.9880777597427368, + "val/ratio_var": 0.0001012428620015271 + }, + { + "episode": 7504, + "epoch": 0.378989898989899, + "eps": 0, + "loss/policy_avg": 0.11465664207935333, + "loss/value_avg": 0.2218267023563385, + "lr": 7.336343141365311e-06, + "objective/entropy": -113.41480255126953, + "objective/kl": 28.445619583129883, + "objective/non_score_reward": -1.4222809076309204, + "objective/rlhf_reward": 10.585531234741211, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0010699313133955002, + "policy/clipfrac_avg": 0.021148769184947014, + "policy/entropy_avg": 0.1771717071533203, + "step": 469, + "val/clipfrac_avg": 0.05555972456932068, + "val/num_eos_tokens": 301, + "val/ratio": 0.9880005121231079, + "val/ratio_var": 5.9720714489230886e-05 + }, + { + "episode": 7520, + "epoch": 0.3797979797979798, + "eps": 0, + "loss/policy_avg": -0.16887971758842468, + "loss/value_avg": 1.1260387897491455, + "lr": 7.324525590765037e-06, + "objective/entropy": -66.94224548339844, + "objective/kl": 24.17656898498535, + "objective/non_score_reward": -1.2088284492492676, + "objective/rlhf_reward": 12.02554702758789, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000874909688718617, + "policy/clipfrac_avg": 0.007139209192246199, + "policy/entropy_avg": 0.18168145418167114, + "step": 470, + "val/clipfrac_avg": 0.13150881230831146, + "val/num_eos_tokens": 267, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.2844137826468796e-05 + }, + { + "episode": 7536, + "epoch": 0.3806060606060606, + "eps": 0, + "loss/policy_avg": -0.1819142997264862, + "loss/value_avg": 0.7196761965751648, + "lr": 7.312691451204178e-06, + "objective/entropy": -28.33433723449707, + "objective/kl": 17.821823120117188, + "objective/non_score_reward": -0.8910911083221436, + "objective/rlhf_reward": 10.921408653259277, + "objective/scores": 11.8125, + "policy/approxkl_avg": 0.0010049407137557864, + "policy/clipfrac_avg": 0.0061156535521149635, + "policy/entropy_avg": 0.18752670288085938, + "step": 471, + "val/clipfrac_avg": 0.18608367443084717, + "val/num_eos_tokens": 99, + "val/ratio": 0.9881096482276917, + "val/ratio_var": 4.828600867767818e-05 + }, + { + "episode": 7552, + "epoch": 0.3814141414141414, + "eps": 0, + "loss/policy_avg": -0.12739495933055878, + "loss/value_avg": 0.6806440949440002, + "lr": 7.300840807136983e-06, + "objective/entropy": -104.83464050292969, + "objective/kl": 29.089664459228516, + "objective/non_score_reward": -1.454483151435852, + "objective/rlhf_reward": 10.115829467773438, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0008825187105685472, + "policy/clipfrac_avg": 0.01230642944574356, + "policy/entropy_avg": 0.16717922687530518, + "step": 472, + "val/clipfrac_avg": 0.1584380865097046, + "val/num_eos_tokens": 160, + "val/ratio": 0.989440381526947, + "val/ratio_var": 6.354587821988389e-05 + }, + { + "episode": 7568, + "epoch": 0.38222222222222224, + "eps": 0, + "loss/policy_avg": 0.013141915202140808, + "loss/value_avg": 1.9222874641418457, + "lr": 7.288973743135495e-06, + "objective/entropy": -16.53008270263672, + "objective/kl": 27.92177963256836, + "objective/non_score_reward": -1.39608895778656, + "objective/rlhf_reward": 11.994535446166992, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0012045095209032297, + "policy/clipfrac_avg": 0.009494627825915813, + "policy/entropy_avg": 0.22336864471435547, + "step": 473, + "val/clipfrac_avg": 0.10058106482028961, + "val/num_eos_tokens": 123, + "val/ratio": 0.985871434211731, + "val/ratio_var": 6.286393181653693e-05 + }, + { + "episode": 7584, + "epoch": 0.38303030303030305, + "eps": 0, + "loss/policy_avg": -0.28272491693496704, + "loss/value_avg": 0.5360223054885864, + "lr": 7.277090343888931e-06, + "objective/entropy": -98.81828308105469, + "objective/kl": 33.50300979614258, + "objective/non_score_reward": -1.6751505136489868, + "objective/rlhf_reward": 11.567037582397461, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.000829789787530899, + "policy/clipfrac_avg": 0.008850395679473877, + "policy/entropy_avg": 0.1538858413696289, + "step": 474, + "val/clipfrac_avg": 0.19075022637844086, + "val/num_eos_tokens": 412, + "val/ratio": 0.9900673031806946, + "val/ratio_var": 5.3723222663393244e-05 + }, + { + "episode": 7600, + "epoch": 0.3838383838383838, + "eps": 0, + "loss/policy_avg": -0.2495127022266388, + "loss/value_avg": 0.3245130181312561, + "lr": 7.265190694203086e-06, + "objective/entropy": -70.26910400390625, + "objective/kl": 32.43540573120117, + "objective/non_score_reward": -1.6217703819274902, + "objective/rlhf_reward": 11.596979141235352, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009185326634906232, + "policy/clipfrac_avg": 0.013037210330367088, + "policy/entropy_avg": 0.1732182502746582, + "step": 475, + "val/clipfrac_avg": 0.0755743458867073, + "val/num_eos_tokens": 391, + "val/ratio": 0.9902758598327637, + "val/ratio_var": 6.952504918444902e-05 + }, + { + "episode": 7616, + "epoch": 0.3846464646464646, + "eps": 0, + "loss/policy_avg": -9.876303374767303e-05, + "loss/value_avg": 0.577019453048706, + "lr": 7.253274878999728e-06, + "objective/entropy": -75.98827362060547, + "objective/kl": 33.66416931152344, + "objective/non_score_reward": -1.6832083463668823, + "objective/rlhf_reward": 10.512104034423828, + "objective/scores": 12.1875, + "policy/approxkl_avg": 0.0009120008908212185, + "policy/clipfrac_avg": 0.007840285077691078, + "policy/entropy_avg": 0.14813995361328125, + "step": 476, + "val/clipfrac_avg": 0.17109884321689606, + "val/num_eos_tokens": 441, + "val/ratio": 0.9897706508636475, + "val/ratio_var": 8.284224895760417e-05 + }, + { + "episode": 7632, + "epoch": 0.38545454545454544, + "eps": 0, + "loss/policy_avg": -0.05202428251504898, + "loss/value_avg": 0.769780158996582, + "lr": 7.241342983315985e-06, + "objective/entropy": -107.81949615478516, + "objective/kl": 30.798736572265625, + "objective/non_score_reward": -1.539936900138855, + "objective/rlhf_reward": 10.460063934326172, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0008913679048418999, + "policy/clipfrac_avg": 0.014285984449088573, + "policy/entropy_avg": 0.14296913146972656, + "step": 477, + "val/clipfrac_avg": 0.13049724698066711, + "val/num_eos_tokens": 247, + "val/ratio": 0.9915003180503845, + "val/ratio_var": 3.443012246862054e-05 + }, + { + "episode": 7648, + "epoch": 0.38626262626262625, + "eps": 0, + "loss/policy_avg": -0.12627719342708588, + "loss/value_avg": 0.489132285118103, + "lr": 7.229395092303753e-06, + "objective/entropy": -69.23106384277344, + "objective/kl": 19.90038299560547, + "objective/non_score_reward": -0.9950191974639893, + "objective/rlhf_reward": 13.98935604095459, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008611504454165697, + "policy/clipfrac_avg": 0.008371494710445404, + "policy/entropy_avg": 0.1330699920654297, + "step": 478, + "val/clipfrac_avg": 0.08708248287439346, + "val/num_eos_tokens": 13, + "val/ratio": 0.9916375875473022, + "val/ratio_var": 4.943081512465142e-05 + }, + { + "episode": 7664, + "epoch": 0.38707070707070707, + "eps": 0, + "loss/policy_avg": 0.13281303644180298, + "loss/value_avg": 1.582331657409668, + "lr": 7.217431291229068e-06, + "objective/entropy": -116.11354064941406, + "objective/kl": 26.035850524902344, + "objective/non_score_reward": -1.3017925024032593, + "objective/rlhf_reward": 10.33102035522461, + "objective/scores": 11.625, + "policy/approxkl_avg": 0.0008872399921528995, + "policy/clipfrac_avg": 0.012195098213851452, + "policy/entropy_avg": 0.1528162956237793, + "step": 479, + "val/clipfrac_avg": 0.13728481531143188, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894596338272095, + "val/ratio_var": 9.663304081186652e-05 + }, + { + "episode": 7680, + "epoch": 0.3878787878787879, + "eps": 0, + "loss/policy_avg": 1.0533212423324585, + "loss/value_avg": 1.3323712348937988, + "lr": 7.205451665471515e-06, + "objective/entropy": -102.0567626953125, + "objective/kl": 28.44430160522461, + "objective/non_score_reward": -1.422215223312378, + "objective/rlhf_reward": 10.90591049194336, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007429048418998718, + "policy/clipfrac_avg": 0.013889473862946033, + "policy/entropy_avg": 0.16810226440429688, + "step": 480, + "val/clipfrac_avg": 0.10630576312541962, + "val/num_eos_tokens": 276, + "val/ratio": 0.9902744293212891, + "val/ratio_var": 4.4153079215902835e-05 + }, + { + "episode": 7696, + "epoch": 0.3886868686868687, + "eps": 0, + "loss/policy_avg": -0.07577499747276306, + "loss/value_avg": 0.18845658004283905, + "lr": 7.193456300523607e-06, + "objective/entropy": -135.23007202148438, + "objective/kl": 34.4516716003418, + "objective/non_score_reward": -1.7225836515426636, + "objective/rlhf_reward": 10.222728729248047, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0011850697919726372, + "policy/clipfrac_avg": 0.011513225734233856, + "policy/entropy_avg": 0.1882985234260559, + "step": 481, + "val/clipfrac_avg": 0.017498323693871498, + "val/num_eos_tokens": 1018, + "val/ratio": 0.9870139360427856, + "val/ratio_var": 6.371740892063826e-05 + }, + { + "episode": 7712, + "epoch": 0.3894949494949495, + "eps": 0, + "loss/policy_avg": 0.030544549226760864, + "loss/value_avg": 0.3417617380619049, + "lr": 7.181445281990182e-06, + "objective/entropy": -64.48172760009766, + "objective/kl": 30.676597595214844, + "objective/non_score_reward": -1.533829927444458, + "objective/rlhf_reward": 10.427107810974121, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0008949321345426142, + "policy/clipfrac_avg": 0.010553957894444466, + "policy/entropy_avg": 0.17522239685058594, + "step": 482, + "val/clipfrac_avg": 0.055152617394924164, + "val/num_eos_tokens": 151, + "val/ratio": 0.9891567230224609, + "val/ratio_var": 4.994387927581556e-05 + }, + { + "episode": 7728, + "epoch": 0.3903030303030303, + "eps": 0, + "loss/policy_avg": -0.09279122203588486, + "loss/value_avg": 0.98249751329422, + "lr": 7.169418695587791e-06, + "objective/entropy": -52.11445617675781, + "objective/kl": 37.32021713256836, + "objective/non_score_reward": -1.8660109043121338, + "objective/rlhf_reward": 10.141801834106445, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0009162778733298182, + "policy/clipfrac_avg": 0.0067903571762144566, + "policy/entropy_avg": 0.19576644897460938, + "step": 483, + "val/clipfrac_avg": 0.06959788501262665, + "val/num_eos_tokens": 63, + "val/ratio": 0.9883674383163452, + "val/ratio_var": 5.329724444891326e-05 + }, + { + "episode": 7744, + "epoch": 0.39111111111111113, + "eps": 0, + "loss/policy_avg": -0.170138418674469, + "loss/value_avg": 0.3470100164413452, + "lr": 7.157376627144084e-06, + "objective/entropy": -79.94268798828125, + "objective/kl": 40.677001953125, + "objective/non_score_reward": -2.0338501930236816, + "objective/rlhf_reward": 11.005212783813477, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009370314655825496, + "policy/clipfrac_avg": 0.00797116756439209, + "policy/entropy_avg": 0.17804527282714844, + "step": 484, + "val/clipfrac_avg": 0.05036664009094238, + "val/num_eos_tokens": 591, + "val/ratio": 0.9882063865661621, + "val/ratio_var": 3.965401992900297e-05 + }, + { + "episode": 7760, + "epoch": 0.39191919191919194, + "eps": 0, + "loss/policy_avg": 0.08009683340787888, + "loss/value_avg": 0.8802474737167358, + "lr": 7.145319162597196e-06, + "objective/entropy": -111.34992218017578, + "objective/kl": 33.60002899169922, + "objective/non_score_reward": -1.6800014972686768, + "objective/rlhf_reward": 9.390310287475586, + "objective/scores": 11.0625, + "policy/approxkl_avg": 0.0007860509213060141, + "policy/clipfrac_avg": 0.0074859317392110825, + "policy/entropy_avg": 0.15513873100280762, + "step": 485, + "val/clipfrac_avg": 0.1623341590166092, + "val/num_eos_tokens": 219, + "val/ratio": 0.9912385940551758, + "val/ratio_var": 4.484504825086333e-05 + }, + { + "episode": 7776, + "epoch": 0.3927272727272727, + "eps": 0, + "loss/policy_avg": 0.31785762310028076, + "loss/value_avg": 0.5920059680938721, + "lr": 7.1332463879951404e-06, + "objective/entropy": -130.91871643066406, + "objective/kl": 34.018707275390625, + "objective/non_score_reward": -1.7009353637695312, + "objective/rlhf_reward": 10.592033386230469, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0007868118118494749, + "policy/clipfrac_avg": 0.011621111072599888, + "policy/entropy_avg": 0.1484971046447754, + "step": 486, + "val/clipfrac_avg": 0.08654516935348511, + "val/num_eos_tokens": 387, + "val/ratio": 0.9907491207122803, + "val/ratio_var": 4.03225640184246e-05 + }, + { + "episode": 7792, + "epoch": 0.3935353535353535, + "eps": 0, + "loss/policy_avg": 0.5671185255050659, + "loss/value_avg": 0.6217664480209351, + "lr": 7.121158389495187e-06, + "objective/entropy": -101.36183166503906, + "objective/kl": 30.16710090637207, + "objective/non_score_reward": -1.5083551406860352, + "objective/rlhf_reward": 10.515082359313965, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011175391264259815, + "policy/clipfrac_avg": 0.02089299075305462, + "policy/entropy_avg": 0.17551612854003906, + "step": 487, + "val/clipfrac_avg": 0.11080241203308105, + "val/num_eos_tokens": 686, + "val/ratio": 0.9885724186897278, + "val/ratio_var": 5.202844113227911e-05 + }, + { + "episode": 7808, + "epoch": 0.39434343434343433, + "eps": 0, + "loss/policy_avg": -0.12260816991329193, + "loss/value_avg": 0.37460604310035706, + "lr": 7.109055253363253e-06, + "objective/entropy": -156.5941925048828, + "objective/kl": 34.61153793334961, + "objective/non_score_reward": -1.7305768728256226, + "objective/rlhf_reward": 10.26161003112793, + "objective/scores": 12.0, + "policy/approxkl_avg": 0.0011252043768763542, + "policy/clipfrac_avg": 0.006505176424980164, + "policy/entropy_avg": 0.21551895141601562, + "step": 488, + "val/clipfrac_avg": 0.07414089143276215, + "val/num_eos_tokens": 29, + "val/ratio": 0.9872661828994751, + "val/ratio_var": 7.018823816906661e-05 + }, + { + "episode": 7824, + "epoch": 0.39515151515151514, + "eps": 0, + "loss/policy_avg": -0.03806200996041298, + "loss/value_avg": 0.5151583552360535, + "lr": 7.096937065973285e-06, + "objective/entropy": -90.52662658691406, + "objective/kl": 35.93262481689453, + "objective/non_score_reward": -1.796631097793579, + "objective/rlhf_reward": 9.008056640625, + "objective/scores": 10.8125, + "policy/approxkl_avg": 0.00089329166803509, + "policy/clipfrac_avg": 0.007040734402835369, + "policy/entropy_avg": 0.15955352783203125, + "step": 489, + "val/clipfrac_avg": 0.09701868891716003, + "val/num_eos_tokens": 121, + "val/ratio": 0.990253210067749, + "val/ratio_var": 5.959846021141857e-05 + }, + { + "episode": 7840, + "epoch": 0.39595959595959596, + "eps": 0, + "loss/policy_avg": 0.18260058760643005, + "loss/value_avg": 1.1594921350479126, + "lr": 7.084803913806642e-06, + "objective/entropy": -107.76602172851562, + "objective/kl": 35.315399169921875, + "objective/non_score_reward": -1.7657699584960938, + "objective/rlhf_reward": 9.812355041503906, + "objective/scores": 11.5625, + "policy/approxkl_avg": 0.0009423610754311085, + "policy/clipfrac_avg": 0.011315671727061272, + "policy/entropy_avg": 0.15435028076171875, + "step": 490, + "val/clipfrac_avg": 0.21639129519462585, + "val/num_eos_tokens": 486, + "val/ratio": 0.9888960123062134, + "val/ratio_var": 6.884950562380254e-05 + }, + { + "episode": 7856, + "epoch": 0.39676767676767677, + "eps": 0, + "loss/policy_avg": -0.09957186132669449, + "loss/value_avg": 0.7260873317718506, + "lr": 7.072655883451478e-06, + "objective/entropy": -41.985530853271484, + "objective/kl": 33.98522186279297, + "objective/non_score_reward": -1.6992610692977905, + "objective/rlhf_reward": 12.589801788330078, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0010092597221955657, + "policy/clipfrac_avg": 0.01904710754752159, + "policy/entropy_avg": 0.15885639190673828, + "step": 491, + "val/clipfrac_avg": 0.15300215780735016, + "val/num_eos_tokens": 182, + "val/ratio": 0.9895732402801514, + "val/ratio_var": 4.7087891289265826e-05 + }, + { + "episode": 7872, + "epoch": 0.3975757575757576, + "eps": 0, + "loss/policy_avg": -0.07856579124927521, + "loss/value_avg": 0.4049687385559082, + "lr": 7.060493061602128e-06, + "objective/entropy": -34.535614013671875, + "objective/kl": 24.98674964904785, + "objective/non_score_reward": -1.2493375539779663, + "objective/rlhf_reward": 11.360037803649902, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001240122364833951, + "policy/clipfrac_avg": 0.014498218894004822, + "policy/entropy_avg": 0.1824660301208496, + "step": 492, + "val/clipfrac_avg": 0.07531505078077316, + "val/num_eos_tokens": 198, + "val/ratio": 0.9880272150039673, + "val/ratio_var": 9.475303522776812e-05 + }, + { + "episode": 7888, + "epoch": 0.3983838383838384, + "eps": 0, + "loss/policy_avg": 0.06024222820997238, + "loss/value_avg": 0.33333200216293335, + "lr": 7.048315535058485e-06, + "objective/entropy": -74.60337829589844, + "objective/kl": 37.4053955078125, + "objective/non_score_reward": -1.870269775390625, + "objective/rlhf_reward": 11.590667724609375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008902495028451085, + "policy/clipfrac_avg": 0.008073626086115837, + "policy/entropy_avg": 0.16165542602539062, + "step": 493, + "val/clipfrac_avg": 0.055267032235860825, + "val/num_eos_tokens": 447, + "val/ratio": 0.9903206825256348, + "val/ratio_var": 3.266169369453564e-05 + }, + { + "episode": 7904, + "epoch": 0.3991919191919192, + "eps": 0, + "loss/policy_avg": -0.06525195389986038, + "loss/value_avg": 0.6768864393234253, + "lr": 7.036123390725381e-06, + "objective/entropy": -54.44144821166992, + "objective/kl": 37.19384765625, + "objective/non_score_reward": -1.8596923351287842, + "objective/rlhf_reward": 14.319994926452637, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0012663885718211532, + "policy/clipfrac_avg": 0.010278100147843361, + "policy/entropy_avg": 0.19942545890808105, + "step": 494, + "val/clipfrac_avg": 0.3375101685523987, + "val/num_eos_tokens": 5, + "val/ratio": 0.9859856367111206, + "val/ratio_var": 8.737371535971761e-05 + }, + { + "episode": 7920, + "epoch": 0.4, + "eps": 0, + "loss/policy_avg": -0.15978729724884033, + "loss/value_avg": 0.4220675230026245, + "lr": 7.023916715611969e-06, + "objective/entropy": -161.6277618408203, + "objective/kl": 34.13693618774414, + "objective/non_score_reward": -1.7068469524383545, + "objective/rlhf_reward": 11.644716262817383, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0014467749278992414, + "policy/clipfrac_avg": 0.01142089907079935, + "policy/entropy_avg": 0.1384185552597046, + "step": 495, + "val/clipfrac_avg": 0.07222695648670197, + "val/num_eos_tokens": 325, + "val/ratio": 0.989600419998169, + "val/ratio_var": 0.00014253522385843098 + }, + { + "episode": 7936, + "epoch": 0.40080808080808084, + "eps": 0, + "loss/policy_avg": -0.6986067295074463, + "loss/value_avg": 1.3764233589172363, + "lr": 7.011695596831102e-06, + "objective/entropy": -129.1576385498047, + "objective/kl": 33.550636291503906, + "objective/non_score_reward": -1.6775317192077637, + "objective/rlhf_reward": 10.937702178955078, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.001147491973824799, + "policy/clipfrac_avg": 0.017196711152791977, + "policy/entropy_avg": 0.207261323928833, + "step": 496, + "val/clipfrac_avg": 0.18510054051876068, + "val/num_eos_tokens": 428, + "val/ratio": 0.9887476563453674, + "val/ratio_var": 9.582716302247718e-05 + }, + { + "episode": 7952, + "epoch": 0.4016161616161616, + "eps": 0, + "loss/policy_avg": -0.18637818098068237, + "loss/value_avg": 0.7886095643043518, + "lr": 6.999460121598704e-06, + "objective/entropy": -98.12596130371094, + "objective/kl": 32.41982650756836, + "objective/non_score_reward": -1.6209914684295654, + "objective/rlhf_reward": 11.339945793151855, + "objective/scores": 12.9375, + "policy/approxkl_avg": 0.000857113569509238, + "policy/clipfrac_avg": 0.00827649887651205, + "policy/entropy_avg": 0.1563873291015625, + "step": 497, + "val/clipfrac_avg": 0.04437314718961716, + "val/num_eos_tokens": 728, + "val/ratio": 0.9897803664207458, + "val/ratio_var": 3.502453910186887e-05 + }, + { + "episode": 7968, + "epoch": 0.4024242424242424, + "eps": 0, + "loss/policy_avg": 0.5554550290107727, + "loss/value_avg": 0.9846082925796509, + "lr": 6.987210377233165e-06, + "objective/entropy": -148.50958251953125, + "objective/kl": 27.433303833007812, + "objective/non_score_reward": -1.3716652393341064, + "objective/rlhf_reward": 11.151771545410156, + "objective/scores": 12.5, + "policy/approxkl_avg": 0.001251452136784792, + "policy/clipfrac_avg": 0.025119343772530556, + "policy/entropy_avg": 0.08732831478118896, + "step": 498, + "val/clipfrac_avg": 0.09154286980628967, + "val/num_eos_tokens": 47, + "val/ratio": 0.9940745234489441, + "val/ratio_var": 3.3608135709073395e-05 + }, + { + "episode": 7984, + "epoch": 0.4032323232323232, + "eps": 0, + "loss/policy_avg": -0.14422039687633514, + "loss/value_avg": 0.6888911128044128, + "lr": 6.974946451154694e-06, + "objective/entropy": -83.75293731689453, + "objective/kl": 34.421539306640625, + "objective/non_score_reward": -1.7210773229599, + "objective/rlhf_reward": 12.411735534667969, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0008086425950750709, + "policy/clipfrac_avg": 0.015222619287669659, + "policy/entropy_avg": 0.1486673355102539, + "step": 499, + "val/clipfrac_avg": 0.2364070564508438, + "val/num_eos_tokens": 388, + "val/ratio": 0.991155207157135, + "val/ratio_var": 5.660337774315849e-05 + }, + { + "episode": 8000, + "epoch": 0.40404040404040403, + "eps": 0, + "loss/policy_avg": -0.20510384440422058, + "loss/value_avg": 0.7008277177810669, + "lr": 6.962668430884716e-06, + "objective/entropy": -83.42674255371094, + "objective/kl": 28.21258544921875, + "objective/non_score_reward": -1.4106292724609375, + "objective/rlhf_reward": 12.112808227539062, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009975788416340947, + "policy/clipfrac_avg": 0.010807021521031857, + "policy/entropy_avg": 0.15832972526550293, + "step": 500, + "val/clipfrac_avg": 0.08799945563077927, + "val/num_eos_tokens": 98, + "val/ratio": 0.989881157875061, + "val/ratio_var": 5.400492955232039e-05 + }, + { + "episode": 8016, + "epoch": 0.40484848484848485, + "eps": 0, + "loss/policy_avg": 0.11570744216442108, + "loss/value_avg": 1.2827966213226318, + "lr": 6.950376404045235e-06, + "objective/entropy": -98.0992660522461, + "objective/kl": 33.63490295410156, + "objective/non_score_reward": -1.6817452907562256, + "objective/rlhf_reward": 11.169817924499512, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.000961613841354847, + "policy/clipfrac_avg": 0.008026218973100185, + "policy/entropy_avg": 0.16532373428344727, + "step": 501, + "val/clipfrac_avg": 0.17985008656978607, + "val/num_eos_tokens": 104, + "val/ratio": 0.988849937915802, + "val/ratio_var": 0.00011337183968862519 + }, + { + "episode": 8032, + "epoch": 0.40565656565656566, + "eps": 0, + "loss/policy_avg": 0.33835944533348083, + "loss/value_avg": 0.479499876499176, + "lr": 6.938070458358217e-06, + "objective/entropy": -92.8426284790039, + "objective/kl": 36.50762176513672, + "objective/non_score_reward": -1.8253811597824097, + "objective/rlhf_reward": 11.557432174682617, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008573720697313547, + "policy/clipfrac_avg": 0.008269022218883038, + "policy/entropy_avg": 0.14825701713562012, + "step": 502, + "val/clipfrac_avg": 0.021562231704592705, + "val/num_eos_tokens": 38, + "val/ratio": 0.9906221628189087, + "val/ratio_var": 2.253732418466825e-05 + }, + { + "episode": 8048, + "epoch": 0.4064646464646465, + "eps": 0, + "loss/policy_avg": -0.003604922443628311, + "loss/value_avg": 0.28834861516952515, + "lr": 6.925750681644954e-06, + "objective/entropy": -211.23294067382812, + "objective/kl": 42.3128776550293, + "objective/non_score_reward": -2.1156439781188965, + "objective/rlhf_reward": 8.333574295043945, + "objective/scores": 10.4375, + "policy/approxkl_avg": 0.0009327945881523192, + "policy/clipfrac_avg": 0.009784913621842861, + "policy/entropy_avg": 0.1659529209136963, + "step": 503, + "val/clipfrac_avg": 0.08479395508766174, + "val/num_eos_tokens": 600, + "val/ratio": 0.9885441660881042, + "val/ratio_var": 8.161830191966146e-05 + }, + { + "episode": 8064, + "epoch": 0.4072727272727273, + "eps": 0, + "loss/policy_avg": 0.345672607421875, + "loss/value_avg": 0.7462859153747559, + "lr": 6.913417161825449e-06, + "objective/entropy": -71.5333251953125, + "objective/kl": 26.93456268310547, + "objective/non_score_reward": -1.3467280864715576, + "objective/rlhf_reward": 11.856396675109863, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0010130183072760701, + "policy/clipfrac_avg": 0.016173746436834335, + "policy/entropy_avg": 0.18730783462524414, + "step": 504, + "val/clipfrac_avg": 0.06575201451778412, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880872964859009, + "val/ratio_var": 9.104741911869496e-05 + }, + { + "episode": 8080, + "epoch": 0.4080808080808081, + "eps": 0, + "loss/policy_avg": 0.5040425062179565, + "loss/value_avg": 0.5650883913040161, + "lr": 6.90106998691778e-06, + "objective/entropy": -105.62996673583984, + "objective/kl": 35.24448013305664, + "objective/non_score_reward": -1.7622239589691162, + "objective/rlhf_reward": 11.698713302612305, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0007742226589471102, + "policy/clipfrac_avg": 0.01576562412083149, + "policy/entropy_avg": 0.18837833404541016, + "step": 505, + "val/clipfrac_avg": 0.04690374806523323, + "val/num_eos_tokens": 570, + "val/ratio": 0.9907522201538086, + "val/ratio_var": 3.813532384810969e-05 + }, + { + "episode": 8096, + "epoch": 0.4088888888888889, + "eps": 0, + "loss/policy_avg": -0.13703672587871552, + "loss/value_avg": 1.1825928688049316, + "lr": 6.88870924503747e-06, + "objective/entropy": -108.91736602783203, + "objective/kl": 27.575519561767578, + "objective/non_score_reward": -1.378775954246521, + "objective/rlhf_reward": 12.504035949707031, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0015673571033403277, + "policy/clipfrac_avg": 0.017193179577589035, + "policy/entropy_avg": 0.14080572128295898, + "step": 506, + "val/clipfrac_avg": 0.10861704498529434, + "val/num_eos_tokens": 979, + "val/ratio": 0.9917985200881958, + "val/ratio_var": 0.00015174220607150346 + }, + { + "episode": 8112, + "epoch": 0.40969696969696967, + "eps": 0, + "loss/policy_avg": -0.4995970129966736, + "loss/value_avg": 1.295984148979187, + "lr": 6.876335024396872e-06, + "objective/entropy": -138.9822998046875, + "objective/kl": 27.58624839782715, + "objective/non_score_reward": -1.3793123960494995, + "objective/rlhf_reward": 11.394125938415527, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0006402102299034595, + "policy/clipfrac_avg": 0.012429703958332539, + "policy/entropy_avg": 0.10546708106994629, + "step": 507, + "val/clipfrac_avg": 0.27246493101119995, + "val/num_eos_tokens": 254, + "val/ratio": 0.9938684701919556, + "val/ratio_var": 3.623984594014473e-05 + }, + { + "episode": 8128, + "epoch": 0.4105050505050505, + "eps": 0, + "loss/policy_avg": 0.02236700803041458, + "loss/value_avg": 1.0374104976654053, + "lr": 6.863947413304521e-06, + "objective/entropy": -50.943092346191406, + "objective/kl": 24.472135543823242, + "objective/non_score_reward": -1.223606824874878, + "objective/rlhf_reward": 11.54201889038086, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0019272379577159882, + "policy/clipfrac_avg": 0.021972181275486946, + "policy/entropy_avg": 0.12678146362304688, + "step": 508, + "val/clipfrac_avg": 0.12712614238262177, + "val/num_eos_tokens": 35, + "val/ratio": 0.991468071937561, + "val/ratio_var": 3.564015059964731e-05 + }, + { + "episode": 8144, + "epoch": 0.4113131313131313, + "eps": 0, + "loss/policy_avg": -0.09880562126636505, + "loss/value_avg": 0.7543489933013916, + "lr": 6.8515465001645184e-06, + "objective/entropy": -138.50244140625, + "objective/kl": 32.953182220458984, + "objective/non_score_reward": -1.6476593017578125, + "objective/rlhf_reward": 11.125778198242188, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009498810395598412, + "policy/clipfrac_avg": 0.015503231436014175, + "policy/entropy_avg": 0.13935184478759766, + "step": 509, + "val/clipfrac_avg": 0.08852463960647583, + "val/num_eos_tokens": 273, + "val/ratio": 0.9911839962005615, + "val/ratio_var": 4.137688301852904e-05 + }, + { + "episode": 8160, + "epoch": 0.4121212121212121, + "eps": 0, + "loss/policy_avg": 0.04148940369486809, + "loss/value_avg": 0.4635436236858368, + "lr": 6.839132373475894e-06, + "objective/entropy": -83.92607116699219, + "objective/kl": 32.373985290527344, + "objective/non_score_reward": -1.618699312210083, + "objective/rlhf_reward": 9.771925926208496, + "objective/scores": 11.375, + "policy/approxkl_avg": 0.000855033693369478, + "policy/clipfrac_avg": 0.006832615472376347, + "policy/entropy_avg": 0.14737749099731445, + "step": 510, + "val/clipfrac_avg": 0.1661660373210907, + "val/num_eos_tokens": 115, + "val/ratio": 0.9900499582290649, + "val/ratio_var": 5.550993591896258e-05 + }, + { + "episode": 8176, + "epoch": 0.4129292929292929, + "eps": 0, + "loss/policy_avg": -0.35513174533843994, + "loss/value_avg": 0.916961669921875, + "lr": 6.8267051218319766e-06, + "objective/entropy": -101.56571960449219, + "objective/kl": 31.993003845214844, + "objective/non_score_reward": -1.5996501445770264, + "objective/rlhf_reward": 11.556599617004395, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0008088398608379066, + "policy/clipfrac_avg": 0.014578074216842651, + "policy/entropy_avg": 0.16936588287353516, + "step": 511, + "val/clipfrac_avg": 0.14982202649116516, + "val/num_eos_tokens": 126, + "val/ratio": 0.9902456998825073, + "val/ratio_var": 6.044187830411829e-05 + }, + { + "episode": 8192, + "epoch": 0.41373737373737374, + "eps": 0, + "loss/policy_avg": 0.004593394696712494, + "loss/value_avg": 0.6813367009162903, + "lr": 6.814264833919758e-06, + "objective/entropy": -98.40760803222656, + "objective/kl": 28.136348724365234, + "objective/non_score_reward": -1.4068175554275513, + "objective/rlhf_reward": 10.440837860107422, + "objective/scores": 11.875, + "policy/approxkl_avg": 0.0008492213673889637, + "policy/clipfrac_avg": 0.008400412276387215, + "policy/entropy_avg": 0.1722790002822876, + "step": 512, + "val/clipfrac_avg": 0.08613656461238861, + "val/num_eos_tokens": 52, + "val/ratio": 0.9898751974105835, + "val/ratio_var": 4.2340398067608476e-05 + }, + { + "episode": 8208, + "epoch": 0.41454545454545455, + "eps": 0, + "loss/policy_avg": 0.09692560136318207, + "loss/value_avg": 0.45964065194129944, + "lr": 6.801811598519268e-06, + "objective/entropy": -121.57412719726562, + "objective/kl": 32.61389923095703, + "objective/non_score_reward": -1.6306949853897095, + "objective/rlhf_reward": 10.650554656982422, + "objective/scores": 12.25, + "policy/approxkl_avg": 0.0007619216339662671, + "policy/clipfrac_avg": 0.008247134275734425, + "policy/entropy_avg": 0.1289653778076172, + "step": 513, + "val/clipfrac_avg": 0.13378626108169556, + "val/num_eos_tokens": 9, + "val/ratio": 0.9914026260375977, + "val/ratio_var": 6.937953730812296e-05 + }, + { + "episode": 8224, + "epoch": 0.41535353535353536, + "eps": 0, + "loss/policy_avg": -0.16650214791297913, + "loss/value_avg": 0.5488893985748291, + "lr": 6.789345504502932e-06, + "objective/entropy": -133.54974365234375, + "objective/kl": 32.966880798339844, + "objective/non_score_reward": -1.6483440399169922, + "objective/rlhf_reward": 13.601655960083008, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008372896118089557, + "policy/clipfrac_avg": 0.01306083146482706, + "policy/entropy_avg": 0.1169576644897461, + "step": 514, + "val/clipfrac_avg": 0.20459802448749542, + "val/num_eos_tokens": 21, + "val/ratio": 0.9914659261703491, + "val/ratio_var": 3.493462281767279e-05 + }, + { + "episode": 8240, + "epoch": 0.4161616161616162, + "eps": 0, + "loss/policy_avg": 0.025405913591384888, + "loss/value_avg": 0.4576077163219452, + "lr": 6.7768666408349445e-06, + "objective/entropy": -105.05879211425781, + "objective/kl": 31.566425323486328, + "objective/non_score_reward": -1.5783213376998901, + "objective/rlhf_reward": 10.85136604309082, + "objective/scores": 12.4375, + "policy/approxkl_avg": 0.0009135222062468529, + "policy/clipfrac_avg": 0.011635120026767254, + "policy/entropy_avg": 0.16112518310546875, + "step": 515, + "val/clipfrac_avg": 0.12827220559120178, + "val/num_eos_tokens": 416, + "val/ratio": 0.9887697696685791, + "val/ratio_var": 4.425182851264253e-05 + }, + { + "episode": 8256, + "epoch": 0.416969696969697, + "eps": 0, + "loss/policy_avg": 0.050351403653621674, + "loss/value_avg": 0.589043915271759, + "lr": 6.764375096570628e-06, + "objective/entropy": -104.26789855957031, + "objective/kl": 32.33205032348633, + "objective/non_score_reward": -1.6166024208068848, + "objective/rlhf_reward": 10.484960556030273, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0010946225374937057, + "policy/clipfrac_avg": 0.009625648148357868, + "policy/entropy_avg": 0.1612253189086914, + "step": 516, + "val/clipfrac_avg": 0.11179494857788086, + "val/num_eos_tokens": 386, + "val/ratio": 0.9899939298629761, + "val/ratio_var": 6.829272751929238e-05 + }, + { + "episode": 8272, + "epoch": 0.4177777777777778, + "eps": 0, + "loss/policy_avg": 0.016396060585975647, + "loss/value_avg": 2.60082745552063, + "lr": 6.7518709608558e-06, + "objective/entropy": -76.06773376464844, + "objective/kl": 33.561153411865234, + "objective/non_score_reward": -1.6780576705932617, + "objective/rlhf_reward": 11.603193283081055, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0009749201126396656, + "policy/clipfrac_avg": 0.018156636506319046, + "policy/entropy_avg": 0.19376277923583984, + "step": 517, + "val/clipfrac_avg": 0.12106801569461823, + "val/num_eos_tokens": 163, + "val/ratio": 0.9880164861679077, + "val/ratio_var": 5.697306551155634e-05 + }, + { + "episode": 8288, + "epoch": 0.41858585858585856, + "eps": 0, + "loss/policy_avg": -0.07019297033548355, + "loss/value_avg": 0.5070562958717346, + "lr": 6.739354322926136e-06, + "objective/entropy": -17.734128952026367, + "objective/kl": 27.766971588134766, + "objective/non_score_reward": -1.3883485794067383, + "objective/rlhf_reward": 12.830401420593262, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008711587870493531, + "policy/clipfrac_avg": 0.004499074071645737, + "policy/entropy_avg": 0.16478729248046875, + "step": 518, + "val/clipfrac_avg": 0.1293402910232544, + "val/num_eos_tokens": 145, + "val/ratio": 0.9889296293258667, + "val/ratio_var": 3.157386163366027e-05 + }, + { + "episode": 8304, + "epoch": 0.4193939393939394, + "eps": 0, + "loss/policy_avg": -0.11896735429763794, + "loss/value_avg": 0.35987716913223267, + "lr": 6.726825272106539e-06, + "objective/entropy": -62.78154373168945, + "objective/kl": 31.432270050048828, + "objective/non_score_reward": -1.5716133117675781, + "objective/rlhf_reward": 12.311199188232422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008107471512630582, + "policy/clipfrac_avg": 0.009700633585453033, + "policy/entropy_avg": 0.12371361255645752, + "step": 519, + "val/clipfrac_avg": 0.12159738689661026, + "val/num_eos_tokens": 365, + "val/ratio": 0.9906887412071228, + "val/ratio_var": 3.533386188792065e-05 + }, + { + "episode": 8320, + "epoch": 0.4202020202020202, + "eps": 0, + "loss/policy_avg": -0.0776340663433075, + "loss/value_avg": 0.3134997487068176, + "lr": 6.714283897810488e-06, + "objective/entropy": -86.20362854003906, + "objective/kl": 32.716697692871094, + "objective/non_score_reward": -1.6358349323272705, + "objective/rlhf_reward": 12.606351852416992, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0021235994063317776, + "policy/clipfrac_avg": 0.015771860256791115, + "policy/entropy_avg": 0.1624317169189453, + "step": 520, + "val/clipfrac_avg": 0.05747883766889572, + "val/num_eos_tokens": 30, + "val/ratio": 0.989377498626709, + "val/ratio_var": 5.577542833634652e-05 + }, + { + "episode": 8336, + "epoch": 0.421010101010101, + "eps": 0, + "loss/policy_avg": -0.464477002620697, + "loss/value_avg": 1.1038016080856323, + "lr": 6.701730289539417e-06, + "objective/entropy": -131.93936157226562, + "objective/kl": 23.357372283935547, + "objective/non_score_reward": -1.1678686141967773, + "objective/rlhf_reward": 12.738380432128906, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0008331306744366884, + "policy/clipfrac_avg": 0.018915321677923203, + "policy/entropy_avg": 0.14388728141784668, + "step": 521, + "val/clipfrac_avg": 0.1614842414855957, + "val/num_eos_tokens": 377, + "val/ratio": 0.9908460378646851, + "val/ratio_var": 6.456010305555537e-05 + }, + { + "episode": 8352, + "epoch": 0.4218181818181818, + "eps": 0, + "loss/policy_avg": -0.130820631980896, + "loss/value_avg": 0.6008985042572021, + "lr": 6.689164536882059e-06, + "objective/entropy": -114.08953094482422, + "objective/kl": 27.40955352783203, + "objective/non_score_reward": -1.3704776763916016, + "objective/rlhf_reward": 11.652959823608398, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.000854157842695713, + "policy/clipfrac_avg": 0.015246449038386345, + "policy/entropy_avg": 0.1589183807373047, + "step": 522, + "val/clipfrac_avg": 0.08382974565029144, + "val/num_eos_tokens": 337, + "val/ratio": 0.9912443161010742, + "val/ratio_var": 5.435436833067797e-05 + }, + { + "episode": 8368, + "epoch": 0.4226262626262626, + "eps": 0, + "loss/policy_avg": -0.1433180719614029, + "loss/value_avg": 0.31795254349708557, + "lr": 6.676586729513823e-06, + "objective/entropy": -129.62985229492188, + "objective/kl": 34.11695098876953, + "objective/non_score_reward": -1.7058476209640503, + "objective/rlhf_reward": 10.251184463500977, + "objective/scores": 11.9375, + "policy/approxkl_avg": 0.0007716008694842458, + "policy/clipfrac_avg": 0.0059022363275289536, + "policy/entropy_avg": 0.14097148180007935, + "step": 523, + "val/clipfrac_avg": 0.14383307099342346, + "val/num_eos_tokens": 34, + "val/ratio": 0.9905247092247009, + "val/ratio_var": 4.685216845246032e-05 + }, + { + "episode": 8384, + "epoch": 0.42343434343434344, + "eps": 0, + "loss/policy_avg": 0.09673983603715897, + "loss/value_avg": 0.37219128012657166, + "lr": 6.6639969571961406e-06, + "objective/entropy": -46.01490783691406, + "objective/kl": 30.304183959960938, + "objective/non_score_reward": -1.5152091979980469, + "objective/rlhf_reward": 13.375415802001953, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009743375121615827, + "policy/clipfrac_avg": 0.008508563973009586, + "policy/entropy_avg": 0.19330501556396484, + "step": 524, + "val/clipfrac_avg": 0.07922305911779404, + "val/num_eos_tokens": 4, + "val/ratio": 0.9889954328536987, + "val/ratio_var": 0.00010105717228725553 + }, + { + "episode": 8400, + "epoch": 0.42424242424242425, + "eps": 0, + "loss/policy_avg": 0.42921364307403564, + "loss/value_avg": 0.41969558596611023, + "lr": 6.651395309775837e-06, + "objective/entropy": -73.01426696777344, + "objective/kl": 26.15176010131836, + "objective/non_score_reward": -1.30758798122406, + "objective/rlhf_reward": 12.481473922729492, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0008940043626353145, + "policy/clipfrac_avg": 0.011357388459146023, + "policy/entropy_avg": 0.18736028671264648, + "step": 525, + "val/clipfrac_avg": 0.04973672330379486, + "val/num_eos_tokens": 32, + "val/ratio": 0.9906824827194214, + "val/ratio_var": 4.821328184334561e-05 + }, + { + "episode": 8416, + "epoch": 0.42505050505050507, + "eps": 0, + "loss/policy_avg": 0.1258348971605301, + "loss/value_avg": 0.46568286418914795, + "lr": 6.638781877184477e-06, + "objective/entropy": -122.70181274414062, + "objective/kl": 29.63160514831543, + "objective/non_score_reward": -1.4815802574157715, + "objective/rlhf_reward": 10.88560676574707, + "objective/scores": 12.375, + "policy/approxkl_avg": 0.0008636523270979524, + "policy/clipfrac_avg": 0.01251271739602089, + "policy/entropy_avg": 0.16792958974838257, + "step": 526, + "val/clipfrac_avg": 0.03258850425481796, + "val/num_eos_tokens": 132, + "val/ratio": 0.9903141260147095, + "val/ratio_var": 3.827890759566799e-05 + }, + { + "episode": 8432, + "epoch": 0.4258585858585859, + "eps": 0, + "loss/policy_avg": 0.0722554475069046, + "loss/value_avg": 1.4936760663986206, + "lr": 6.626156749437736e-06, + "objective/entropy": -157.23638916015625, + "objective/kl": 26.3393611907959, + "objective/non_score_reward": -1.3169679641723633, + "objective/rlhf_reward": 12.40959358215332, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009735558414831758, + "policy/clipfrac_avg": 0.007334539666771889, + "policy/entropy_avg": 0.12781143188476562, + "step": 527, + "val/clipfrac_avg": 0.08778189867734909, + "val/num_eos_tokens": 42, + "val/ratio": 0.993151068687439, + "val/ratio_var": 0.00014699323219247162 + }, + { + "episode": 8448, + "epoch": 0.4266666666666667, + "eps": 0, + "loss/policy_avg": 0.2405271977186203, + "loss/value_avg": 0.30028170347213745, + "lr": 6.6135200166347505e-06, + "objective/entropy": -100.36758422851562, + "objective/kl": 39.47442626953125, + "objective/non_score_reward": -1.9737212657928467, + "objective/rlhf_reward": 10.36221694946289, + "objective/scores": 12.3125, + "policy/approxkl_avg": 0.0009299812954850495, + "policy/clipfrac_avg": 0.011641680262982845, + "policy/entropy_avg": 0.16964960098266602, + "step": 528, + "val/clipfrac_avg": 0.17108477652072906, + "val/num_eos_tokens": 247, + "val/ratio": 0.9882971048355103, + "val/ratio_var": 5.371840961743146e-05 + }, + { + "episode": 8464, + "epoch": 0.42747474747474745, + "eps": 0, + "loss/policy_avg": -0.15250246226787567, + "loss/value_avg": 0.8714890480041504, + "lr": 6.600871768957474e-06, + "objective/entropy": -314.8489990234375, + "objective/kl": 30.68231964111328, + "objective/non_score_reward": -1.5341161489486694, + "objective/rlhf_reward": 11.434633255004883, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0009698778740130365, + "policy/clipfrac_avg": 0.01093447208404541, + "policy/entropy_avg": 0.10747766494750977, + "step": 529, + "val/clipfrac_avg": 0.09076748788356781, + "val/num_eos_tokens": 650, + "val/ratio": 0.9922375679016113, + "val/ratio_var": 6.258123903535306e-05 + }, + { + "episode": 8480, + "epoch": 0.42828282828282827, + "eps": 0, + "loss/policy_avg": -0.35478073358535767, + "loss/value_avg": 0.8044853210449219, + "lr": 6.588212096670037e-06, + "objective/entropy": -109.72494506835938, + "objective/kl": 41.18302917480469, + "objective/non_score_reward": -2.0591516494750977, + "objective/rlhf_reward": 11.933036804199219, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007708189077675343, + "policy/clipfrac_avg": 0.005906249396502972, + "policy/entropy_avg": 0.14908885955810547, + "step": 530, + "val/clipfrac_avg": 0.06992866098880768, + "val/num_eos_tokens": 571, + "val/ratio": 0.9903333187103271, + "val/ratio_var": 7.135802297852933e-05 + }, + { + "episode": 8496, + "epoch": 0.4290909090909091, + "eps": 0, + "loss/policy_avg": 0.08454768359661102, + "loss/value_avg": 0.4138789176940918, + "lr": 6.575541090118105e-06, + "objective/entropy": -76.89524841308594, + "objective/kl": 32.64395523071289, + "objective/non_score_reward": -1.6321979761123657, + "objective/rlhf_reward": 10.934207916259766, + "objective/scores": 12.5625, + "policy/approxkl_avg": 0.0009906484046950936, + "policy/clipfrac_avg": 0.006618197076022625, + "policy/entropy_avg": 0.1799774169921875, + "step": 531, + "val/clipfrac_avg": 0.08779165148735046, + "val/num_eos_tokens": 947, + "val/ratio": 0.9877060651779175, + "val/ratio_var": 5.0242877478012815e-05 + }, + { + "episode": 8512, + "epoch": 0.4298989898989899, + "eps": 0, + "loss/policy_avg": -0.6856231093406677, + "loss/value_avg": 0.972754180431366, + "lr": 6.562858839728224e-06, + "objective/entropy": -62.30128860473633, + "objective/kl": 31.940135955810547, + "objective/non_score_reward": -1.597006916999817, + "objective/rlhf_reward": 12.582681655883789, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010326767805963755, + "policy/clipfrac_avg": 0.005477210506796837, + "policy/entropy_avg": 0.1778457760810852, + "step": 532, + "val/clipfrac_avg": 0.1998615562915802, + "val/num_eos_tokens": 38, + "val/ratio": 0.9881020188331604, + "val/ratio_var": 5.7397228374611586e-05 + }, + { + "episode": 8528, + "epoch": 0.4307070707070707, + "eps": 0, + "loss/policy_avg": -0.00426415354013443, + "loss/value_avg": 0.3097630739212036, + "lr": 6.550165436007185e-06, + "objective/entropy": -143.62107849121094, + "objective/kl": 24.379880905151367, + "objective/non_score_reward": -1.218994140625, + "objective/rlhf_reward": 12.210693359375, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.0008440241799689829, + "policy/clipfrac_avg": 0.009614679962396622, + "policy/entropy_avg": 0.13739681243896484, + "step": 533, + "val/clipfrac_avg": 0.052517104893922806, + "val/num_eos_tokens": 10, + "val/ratio": 0.9900696277618408, + "val/ratio_var": 6.144143844721839e-05 + }, + { + "episode": 8544, + "epoch": 0.4315151515151515, + "eps": 0, + "loss/policy_avg": -0.09833642840385437, + "loss/value_avg": 0.3559184670448303, + "lr": 6.537460969541378e-06, + "objective/entropy": -39.871124267578125, + "objective/kl": 30.961679458618164, + "objective/non_score_reward": -1.548084020614624, + "objective/rlhf_reward": 13.412853240966797, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0011838905047625303, + "policy/clipfrac_avg": 0.01087412890046835, + "policy/entropy_avg": 0.18686485290527344, + "step": 534, + "val/clipfrac_avg": 0.11940991878509521, + "val/num_eos_tokens": 60, + "val/ratio": 0.9873068928718567, + "val/ratio_var": 5.543338193092495e-05 + }, + { + "episode": 8560, + "epoch": 0.43232323232323233, + "eps": 0, + "loss/policy_avg": 0.18861787021160126, + "loss/value_avg": 0.3709791898727417, + "lr": 6.524745530996137e-06, + "objective/entropy": -58.27792739868164, + "objective/kl": 39.486045837402344, + "objective/non_score_reward": -1.9743022918701172, + "objective/rlhf_reward": 10.916322708129883, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008673222037032247, + "policy/clipfrac_avg": 0.0051139891147613525, + "policy/entropy_avg": 0.1625194549560547, + "step": 535, + "val/clipfrac_avg": 0.19597014784812927, + "val/num_eos_tokens": 189, + "val/ratio": 0.9891222715377808, + "val/ratio_var": 2.865192072931677e-05 + }, + { + "episode": 8576, + "epoch": 0.43313131313131314, + "eps": 0, + "loss/policy_avg": -0.030973181128501892, + "loss/value_avg": 0.6104284524917603, + "lr": 6.512019211115099e-06, + "objective/entropy": -41.29559326171875, + "objective/kl": 28.339763641357422, + "objective/non_score_reward": -1.4169882535934448, + "objective/rlhf_reward": 12.317386627197266, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0007994387997314334, + "policy/clipfrac_avg": 0.01735379919409752, + "policy/entropy_avg": 0.17484283447265625, + "step": 536, + "val/clipfrac_avg": 0.061991170048713684, + "val/num_eos_tokens": 158, + "val/ratio": 0.9900209903717041, + "val/ratio_var": 5.226740540820174e-05 + }, + { + "episode": 8592, + "epoch": 0.43393939393939396, + "eps": 0, + "loss/policy_avg": -0.14375048875808716, + "loss/value_avg": 0.18352828919887543, + "lr": 6.499282100719558e-06, + "objective/entropy": -98.1541976928711, + "objective/kl": 36.661048889160156, + "objective/non_score_reward": -1.833052396774292, + "objective/rlhf_reward": 11.971634864807129, + "objective/scores": 13.8125, + "policy/approxkl_avg": 0.0011100426781922579, + "policy/clipfrac_avg": 0.009958980605006218, + "policy/entropy_avg": 0.14308547973632812, + "step": 537, + "val/clipfrac_avg": 0.05129721760749817, + "val/num_eos_tokens": 479, + "val/ratio": 0.9901905059814453, + "val/ratio_var": 3.793285577557981e-05 + }, + { + "episode": 8608, + "epoch": 0.43474747474747477, + "eps": 0, + "loss/policy_avg": -0.06621083617210388, + "loss/value_avg": 0.621555745601654, + "lr": 6.486534290707815e-06, + "objective/entropy": -113.62753295898438, + "objective/kl": 31.673198699951172, + "objective/non_score_reward": -1.5836598873138428, + "objective/rlhf_reward": 12.431964874267578, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0014390661381185055, + "policy/clipfrac_avg": 0.02696000598371029, + "policy/entropy_avg": 0.13946056365966797, + "step": 538, + "val/clipfrac_avg": 0.10710769891738892, + "val/num_eos_tokens": 342, + "val/ratio": 0.9896073341369629, + "val/ratio_var": 9.375382796861231e-05 + }, + { + "episode": 8624, + "epoch": 0.43555555555555553, + "eps": 0, + "loss/policy_avg": -0.06200864911079407, + "loss/value_avg": 0.3554137051105499, + "lr": 6.473775872054522e-06, + "objective/entropy": -120.77999877929688, + "objective/kl": 34.75250244140625, + "objective/non_score_reward": -1.7376251220703125, + "objective/rlhf_reward": 11.762374877929688, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0009148735553026199, + "policy/clipfrac_avg": 0.005174038931727409, + "policy/entropy_avg": 0.14241600036621094, + "step": 539, + "val/clipfrac_avg": 0.06834189593791962, + "val/num_eos_tokens": 484, + "val/ratio": 0.9897119998931885, + "val/ratio_var": 5.696159860235639e-05 + }, + { + "episode": 8640, + "epoch": 0.43636363636363634, + "eps": 0, + "loss/policy_avg": 0.060983721166849136, + "loss/value_avg": 0.4710575342178345, + "lr": 6.461006935810048e-06, + "objective/entropy": -115.64312744140625, + "objective/kl": 40.74346160888672, + "objective/non_score_reward": -2.03717303276062, + "objective/rlhf_reward": 10.947202682495117, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.0008006396819837391, + "policy/clipfrac_avg": 0.006549537181854248, + "policy/entropy_avg": 0.141357421875, + "step": 540, + "val/clipfrac_avg": 0.15336185693740845, + "val/num_eos_tokens": 624, + "val/ratio": 0.9915748238563538, + "val/ratio_var": 5.314065856509842e-05 + }, + { + "episode": 8656, + "epoch": 0.43717171717171716, + "eps": 0, + "loss/policy_avg": -0.39746806025505066, + "loss/value_avg": 1.7929481267929077, + "lr": 6.448227573099815e-06, + "objective/entropy": -112.73287963867188, + "objective/kl": 31.47679901123047, + "objective/non_score_reward": -1.5738401412963867, + "objective/rlhf_reward": 13.254284858703613, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009382686112076044, + "policy/clipfrac_avg": 0.0064744651317596436, + "policy/entropy_avg": 0.15688323974609375, + "step": 541, + "val/clipfrac_avg": 0.2390729784965515, + "val/num_eos_tokens": 1255, + "val/ratio": 0.9905893802642822, + "val/ratio_var": 9.629622945794836e-05 + }, + { + "episode": 8672, + "epoch": 0.43797979797979797, + "eps": 0, + "loss/policy_avg": -0.2003842294216156, + "loss/value_avg": 0.4203144907951355, + "lr": 6.435437875123658e-06, + "objective/entropy": -104.77660369873047, + "objective/kl": 33.559234619140625, + "objective/non_score_reward": -1.677961826324463, + "objective/rlhf_reward": 13.173601150512695, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0010652798227965832, + "policy/clipfrac_avg": 0.005728135816752911, + "policy/entropy_avg": 0.1696929931640625, + "step": 542, + "val/clipfrac_avg": 0.1355896294116974, + "val/num_eos_tokens": 1683, + "val/ratio": 0.9887447953224182, + "val/ratio_var": 5.705601142835803e-05 + }, + { + "episode": 8688, + "epoch": 0.4387878787878788, + "eps": 0, + "loss/policy_avg": -0.05463821440935135, + "loss/value_avg": 0.33894190192222595, + "lr": 6.4226379331551625e-06, + "objective/entropy": -78.30276489257812, + "objective/kl": 29.884620666503906, + "objective/non_score_reward": -1.4942309856414795, + "objective/rlhf_reward": 13.052643775939941, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008832053281366825, + "policy/clipfrac_avg": 0.010442769154906273, + "policy/entropy_avg": 0.1459188461303711, + "step": 543, + "val/clipfrac_avg": 0.05107077211141586, + "val/num_eos_tokens": 112, + "val/ratio": 0.9910210967063904, + "val/ratio_var": 3.290848690085113e-05 + }, + { + "episode": 8704, + "epoch": 0.4395959595959596, + "eps": 0, + "loss/policy_avg": 0.16157524287700653, + "loss/value_avg": 0.43978190422058105, + "lr": 6.409827838541029e-06, + "objective/entropy": -48.91859817504883, + "objective/kl": 41.26933288574219, + "objective/non_score_reward": -2.063466787338257, + "objective/rlhf_reward": 12.561532974243164, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008429911686107516, + "policy/clipfrac_avg": 0.006084254011511803, + "policy/entropy_avg": 0.15523910522460938, + "step": 544, + "val/clipfrac_avg": 0.0520780012011528, + "val/num_eos_tokens": 150, + "val/ratio": 0.990166187286377, + "val/ratio_var": 4.473194348975085e-05 + }, + { + "episode": 8720, + "epoch": 0.4404040404040404, + "eps": 0, + "loss/policy_avg": -0.15311197936534882, + "loss/value_avg": 0.4987826645374298, + "lr": 6.397007682700407e-06, + "objective/entropy": -105.18928527832031, + "objective/kl": 35.155426025390625, + "objective/non_score_reward": -1.7577712535858154, + "objective/rlhf_reward": 14.875041961669922, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011077180970460176, + "policy/clipfrac_avg": 0.010063378140330315, + "policy/entropy_avg": 0.17020034790039062, + "step": 545, + "val/clipfrac_avg": 0.10631218552589417, + "val/num_eos_tokens": 501, + "val/ratio": 0.9899839758872986, + "val/ratio_var": 7.999869558261707e-05 + }, + { + "episode": 8736, + "epoch": 0.4412121212121212, + "eps": 0, + "loss/policy_avg": 0.1040591150522232, + "loss/value_avg": 0.4214034080505371, + "lr": 6.384177557124247e-06, + "objective/entropy": -31.199787139892578, + "objective/kl": 31.340700149536133, + "objective/non_score_reward": -1.5670350790023804, + "objective/rlhf_reward": 11.190776824951172, + "objective/scores": 12.75, + "policy/approxkl_avg": 0.0009682231466285884, + "policy/clipfrac_avg": 0.006479987874627113, + "policy/entropy_avg": 0.20100021362304688, + "step": 546, + "val/clipfrac_avg": 0.1099347323179245, + "val/num_eos_tokens": 238, + "val/ratio": 0.9890173673629761, + "val/ratio_var": 3.694462066050619e-05 + }, + { + "episode": 8752, + "epoch": 0.44202020202020204, + "eps": 0, + "loss/policy_avg": 0.5359160304069519, + "loss/value_avg": 1.8251168727874756, + "lr": 6.3713375533746525e-06, + "objective/entropy": -39.132110595703125, + "objective/kl": 34.65884780883789, + "objective/non_score_reward": -1.7329423427581787, + "objective/rlhf_reward": 9.946744918823242, + "objective/scores": 11.6875, + "policy/approxkl_avg": 0.000952253001742065, + "policy/clipfrac_avg": 0.01447022333741188, + "policy/entropy_avg": 0.18070626258850098, + "step": 547, + "val/clipfrac_avg": 0.09755296260118484, + "val/num_eos_tokens": 147, + "val/ratio": 0.9890221953392029, + "val/ratio_var": 4.6701326937181875e-05 + }, + { + "episode": 8768, + "epoch": 0.44282828282828285, + "eps": 0, + "loss/policy_avg": 0.003339662216603756, + "loss/value_avg": 0.3004421889781952, + "lr": 6.358487763084219e-06, + "objective/entropy": -175.40753173828125, + "objective/kl": 37.905120849609375, + "objective/non_score_reward": -1.8952560424804688, + "objective/rlhf_reward": 12.608650207519531, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000838284962810576, + "policy/clipfrac_avg": 0.003946049604564905, + "policy/entropy_avg": 0.15326714515686035, + "step": 548, + "val/clipfrac_avg": 0.07305619865655899, + "val/num_eos_tokens": 253, + "val/ratio": 0.9898229837417603, + "val/ratio_var": 4.8889804020291194e-05 + }, + { + "episode": 8784, + "epoch": 0.44363636363636366, + "eps": 0, + "loss/policy_avg": 0.07848238945007324, + "loss/value_avg": 1.0936524868011475, + "lr": 6.345628277955384e-06, + "objective/entropy": -127.72113037109375, + "objective/kl": 31.03716278076172, + "objective/non_score_reward": -1.5518580675125122, + "objective/rlhf_reward": 13.635642051696777, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000858222832903266, + "policy/clipfrac_avg": 0.012858349829912186, + "policy/entropy_avg": 0.1750965118408203, + "step": 549, + "val/clipfrac_avg": 0.10532164573669434, + "val/num_eos_tokens": 707, + "val/ratio": 0.990279495716095, + "val/ratio_var": 5.98018268647138e-05 + }, + { + "episode": 8800, + "epoch": 0.4444444444444444, + "eps": 0, + "loss/policy_avg": -0.13267037272453308, + "loss/value_avg": 0.6979200839996338, + "lr": 6.3327591897597765e-06, + "objective/entropy": -142.31295776367188, + "objective/kl": 30.36469268798828, + "objective/non_score_reward": -1.5182347297668457, + "objective/rlhf_reward": 13.255203247070312, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009378796094097197, + "policy/clipfrac_avg": 0.00806361623108387, + "policy/entropy_avg": 0.1446208953857422, + "step": 550, + "val/clipfrac_avg": 0.21628013253211975, + "val/num_eos_tokens": 1703, + "val/ratio": 0.9899749755859375, + "val/ratio_var": 7.445859955623746e-05 + }, + { + "episode": 8816, + "epoch": 0.44525252525252523, + "eps": 0, + "loss/policy_avg": 0.15575791895389557, + "loss/value_avg": 0.4495770335197449, + "lr": 6.319880590337549e-06, + "objective/entropy": -74.91482543945312, + "objective/kl": 28.549150466918945, + "objective/non_score_reward": -1.4274574518203735, + "objective/rlhf_reward": 13.556917190551758, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0011112524662166834, + "policy/clipfrac_avg": 0.011593078263103962, + "policy/entropy_avg": 0.17023122310638428, + "step": 551, + "val/clipfrac_avg": 0.09938865900039673, + "val/num_eos_tokens": 124, + "val/ratio": 0.9868941903114319, + "val/ratio_var": 0.00012118481390643865 + }, + { + "episode": 8832, + "epoch": 0.44606060606060605, + "eps": 0, + "loss/policy_avg": -0.158098965883255, + "loss/value_avg": 0.5848870277404785, + "lr": 6.306992571596742e-06, + "objective/entropy": -101.52589416503906, + "objective/kl": 38.20920944213867, + "objective/non_score_reward": -1.9104604721069336, + "objective/rlhf_reward": 13.23016357421875, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0010805606143549085, + "policy/clipfrac_avg": 0.012465233914554119, + "policy/entropy_avg": 0.1814594268798828, + "step": 552, + "val/clipfrac_avg": 0.03225297853350639, + "val/num_eos_tokens": 138, + "val/ratio": 0.9882332682609558, + "val/ratio_var": 5.3187635785434395e-05 + }, + { + "episode": 8848, + "epoch": 0.44686868686868686, + "eps": 0, + "loss/policy_avg": 0.11628187447786331, + "loss/value_avg": 0.6404980421066284, + "lr": 6.294095225512604e-06, + "objective/entropy": -113.32449340820312, + "objective/kl": 36.24083709716797, + "objective/non_score_reward": -1.8120418787002563, + "objective/rlhf_reward": 11.742645263671875, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0010494563030079007, + "policy/clipfrac_avg": 0.012232597917318344, + "policy/entropy_avg": 0.16427016258239746, + "step": 553, + "val/clipfrac_avg": 0.14791586995124817, + "val/num_eos_tokens": 22, + "val/ratio": 0.9883185625076294, + "val/ratio_var": 6.705433770548552e-05 + }, + { + "episode": 8864, + "epoch": 0.4476767676767677, + "eps": 0, + "loss/policy_avg": 0.5528974533081055, + "loss/value_avg": 0.9337465167045593, + "lr": 6.281188644126961e-06, + "objective/entropy": -56.46511459350586, + "objective/kl": 36.22610092163086, + "objective/non_score_reward": -1.811305046081543, + "objective/rlhf_reward": 10.868382453918457, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009609910775907338, + "policy/clipfrac_avg": 0.020421186462044716, + "policy/entropy_avg": 0.21524810791015625, + "step": 554, + "val/clipfrac_avg": 0.02570945769548416, + "val/num_eos_tokens": 5, + "val/ratio": 0.9899651408195496, + "val/ratio_var": 2.9964650821057148e-05 + }, + { + "episode": 8880, + "epoch": 0.4484848484848485, + "eps": 0, + "loss/policy_avg": -0.13656526803970337, + "loss/value_avg": 0.38830268383026123, + "lr": 6.268272919547537e-06, + "objective/entropy": -92.14622497558594, + "objective/kl": 43.30995559692383, + "objective/non_score_reward": -2.1654977798461914, + "objective/rlhf_reward": 12.537628173828125, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001519187237136066, + "policy/clipfrac_avg": 0.008950314484536648, + "policy/entropy_avg": 0.16514253616333008, + "step": 555, + "val/clipfrac_avg": 0.12284453213214874, + "val/num_eos_tokens": 564, + "val/ratio": 0.9889045357704163, + "val/ratio_var": 9.022261656355113e-05 + }, + { + "episode": 8896, + "epoch": 0.4492929292929293, + "eps": 0, + "loss/policy_avg": -0.2719430923461914, + "loss/value_avg": 0.5197917222976685, + "lr": 6.255348143947313e-06, + "objective/entropy": -117.01277923583984, + "objective/kl": 42.43487548828125, + "objective/non_score_reward": -2.12174391746521, + "objective/rlhf_reward": 12.104818344116211, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008287889650091529, + "policy/clipfrac_avg": 0.008630437776446342, + "policy/entropy_avg": 0.13199996948242188, + "step": 556, + "val/clipfrac_avg": 0.06423404067754745, + "val/num_eos_tokens": 354, + "val/ratio": 0.9903758764266968, + "val/ratio_var": 5.3244064474711195e-05 + }, + { + "episode": 8912, + "epoch": 0.4501010101010101, + "eps": 0, + "loss/policy_avg": -0.0356672927737236, + "loss/value_avg": 0.18395909667015076, + "lr": 6.242414409563855e-06, + "objective/entropy": -73.03179931640625, + "objective/kl": 42.79301452636719, + "objective/non_score_reward": -2.139650821685791, + "objective/rlhf_reward": 12.75878620147705, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008902574772946537, + "policy/clipfrac_avg": 0.006880009081214666, + "policy/entropy_avg": 0.16189002990722656, + "step": 557, + "val/clipfrac_avg": 0.0390281043946743, + "val/num_eos_tokens": 239, + "val/ratio": 0.9893250465393066, + "val/ratio_var": 4.1720588342286646e-05 + }, + { + "episode": 8928, + "epoch": 0.4509090909090909, + "eps": 0, + "loss/policy_avg": 0.02715390920639038, + "loss/value_avg": 0.41223347187042236, + "lr": 6.229471808698673e-06, + "objective/entropy": -104.41824340820312, + "objective/kl": 44.569541931152344, + "objective/non_score_reward": -2.2284772396087646, + "objective/rlhf_reward": 13.107460021972656, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009092853288166225, + "policy/clipfrac_avg": 0.010719258338212967, + "policy/entropy_avg": 0.13871383666992188, + "step": 558, + "val/clipfrac_avg": 0.03966514393687248, + "val/num_eos_tokens": 642, + "val/ratio": 0.9900314807891846, + "val/ratio_var": 5.0811173423426226e-05 + }, + { + "episode": 8944, + "epoch": 0.45171717171717174, + "eps": 0, + "loss/policy_avg": 0.1378185749053955, + "loss/value_avg": 0.5425667762756348, + "lr": 6.216520433716544e-06, + "objective/entropy": -57.53273010253906, + "objective/kl": 46.451202392578125, + "objective/non_score_reward": -2.3225603103637695, + "objective/rlhf_reward": 11.65400218963623, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009985085343942046, + "policy/clipfrac_avg": 0.009565187618136406, + "policy/entropy_avg": 0.16213226318359375, + "step": 559, + "val/clipfrac_avg": 0.14630571007728577, + "val/num_eos_tokens": 76, + "val/ratio": 0.9897133111953735, + "val/ratio_var": 4.261602953192778e-05 + }, + { + "episode": 8960, + "epoch": 0.45252525252525255, + "eps": 0, + "loss/policy_avg": -0.009004142135381699, + "loss/value_avg": 0.3125573992729187, + "lr": 6.2035603770448664e-06, + "objective/entropy": -166.40460205078125, + "objective/kl": 35.97865676879883, + "objective/non_score_reward": -1.7989329099655151, + "objective/rlhf_reward": 11.099504470825195, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0014841826632618904, + "policy/clipfrac_avg": 0.011655549518764019, + "policy/entropy_avg": 0.15684080123901367, + "step": 560, + "val/clipfrac_avg": 0.06238756328821182, + "val/num_eos_tokens": 381, + "val/ratio": 0.9896181225776672, + "val/ratio_var": 8.536590758012608e-05 + }, + { + "episode": 8976, + "epoch": 0.4533333333333333, + "eps": 0, + "loss/policy_avg": -0.011498738080263138, + "loss/value_avg": 0.5593593716621399, + "lr": 6.1905917311729915e-06, + "objective/entropy": -117.21914672851562, + "objective/kl": 30.727413177490234, + "objective/non_score_reward": -1.5363706350326538, + "objective/rlhf_reward": 11.635503768920898, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.001313843298703432, + "policy/clipfrac_avg": 0.013457929715514183, + "policy/entropy_avg": 0.16321659088134766, + "step": 561, + "val/clipfrac_avg": 0.06608335673809052, + "val/num_eos_tokens": 5, + "val/ratio": 0.9888886213302612, + "val/ratio_var": 6.117957673268393e-05 + }, + { + "episode": 8992, + "epoch": 0.4541414141414141, + "eps": 0, + "loss/policy_avg": -0.0510425865650177, + "loss/value_avg": 0.3040882349014282, + "lr": 6.177614588651572e-06, + "objective/entropy": -31.294681549072266, + "objective/kl": 36.154850006103516, + "objective/non_score_reward": -1.8077425956726074, + "objective/rlhf_reward": 12.184444427490234, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0010273936204612255, + "policy/clipfrac_avg": 0.007020684890449047, + "policy/entropy_avg": 0.1751842498779297, + "step": 562, + "val/clipfrac_avg": 0.06005093827843666, + "val/num_eos_tokens": 162, + "val/ratio": 0.9871533513069153, + "val/ratio_var": 6.729259621351957e-05 + }, + { + "episode": 9008, + "epoch": 0.45494949494949494, + "eps": 0, + "loss/policy_avg": 0.03235238790512085, + "loss/value_avg": 0.7062409520149231, + "lr": 6.164629042091894e-06, + "objective/entropy": -124.56060791015625, + "objective/kl": 36.547935485839844, + "objective/non_score_reward": -1.8273968696594238, + "objective/rlhf_reward": 13.203852653503418, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008937460370361805, + "policy/clipfrac_avg": 0.01175684854388237, + "policy/entropy_avg": 0.13650989532470703, + "step": 563, + "val/clipfrac_avg": 0.08029687404632568, + "val/num_eos_tokens": 503, + "val/ratio": 0.991959810256958, + "val/ratio_var": 6.220470095286146e-05 + }, + { + "episode": 9024, + "epoch": 0.45575757575757575, + "eps": 0, + "loss/policy_avg": 0.0829654112458229, + "loss/value_avg": 0.461310476064682, + "lr": 6.151635184165219e-06, + "objective/entropy": -140.72860717773438, + "objective/kl": 33.58654022216797, + "objective/non_score_reward": -1.679327130317688, + "objective/rlhf_reward": 11.437860488891602, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0007259284611791372, + "policy/clipfrac_avg": 0.008242369629442692, + "policy/entropy_avg": 0.1111001968383789, + "step": 564, + "val/clipfrac_avg": 0.04573877528309822, + "val/num_eos_tokens": 393, + "val/ratio": 0.9929757118225098, + "val/ratio_var": 2.9616816391353495e-05 + }, + { + "episode": 9040, + "epoch": 0.45656565656565656, + "eps": 0, + "loss/policy_avg": -0.13684070110321045, + "loss/value_avg": 0.2872938811779022, + "lr": 6.1386331076021234e-06, + "objective/entropy": -49.62328338623047, + "objective/kl": 38.57415771484375, + "objective/non_score_reward": -1.9287079572677612, + "objective/rlhf_reward": 14.235355377197266, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010976013727486134, + "policy/clipfrac_avg": 0.004873812198638916, + "policy/entropy_avg": 0.19704437255859375, + "step": 565, + "val/clipfrac_avg": 0.21986615657806396, + "val/num_eos_tokens": 90, + "val/ratio": 0.9863287210464478, + "val/ratio_var": 5.253278141026385e-05 + }, + { + "episode": 9056, + "epoch": 0.4573737373737374, + "eps": 0, + "loss/policy_avg": 0.15097515285015106, + "loss/value_avg": 0.8962790966033936, + "lr": 6.125622905191835e-06, + "objective/entropy": -66.99671936035156, + "objective/kl": 40.845550537109375, + "objective/non_score_reward": -2.0422775745391846, + "objective/rlhf_reward": 12.223346710205078, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009532993426546454, + "policy/clipfrac_avg": 0.008950391784310341, + "policy/entropy_avg": 0.15424060821533203, + "step": 566, + "val/clipfrac_avg": 0.16976982355117798, + "val/num_eos_tokens": 139, + "val/ratio": 0.9896418452262878, + "val/ratio_var": 6.317990482784808e-05 + }, + { + "episode": 9072, + "epoch": 0.4581818181818182, + "eps": 0, + "loss/policy_avg": 0.0012879595160484314, + "loss/value_avg": 0.6243962049484253, + "lr": 6.112604669781572e-06, + "objective/entropy": -69.26618957519531, + "objective/kl": 31.181835174560547, + "objective/non_score_reward": -1.5590916872024536, + "objective/rlhf_reward": 11.479970932006836, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0010285003809258342, + "policy/clipfrac_avg": 0.007737594656646252, + "policy/entropy_avg": 0.19235610961914062, + "step": 567, + "val/clipfrac_avg": 0.07829482853412628, + "val/num_eos_tokens": 147, + "val/ratio": 0.9867407083511353, + "val/ratio_var": 7.090938743203878e-05 + }, + { + "episode": 9088, + "epoch": 0.458989898989899, + "eps": 0, + "loss/policy_avg": -0.19695457816123962, + "loss/value_avg": 0.6798470616340637, + "lr": 6.099578494275883e-06, + "objective/entropy": -65.55770874023438, + "objective/kl": 28.36505889892578, + "objective/non_score_reward": -1.418252944946289, + "objective/rlhf_reward": 13.370809555053711, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009267916902899742, + "policy/clipfrac_avg": 0.013094292022287846, + "policy/entropy_avg": 0.15218353271484375, + "step": 568, + "val/clipfrac_avg": 0.11755470186471939, + "val/num_eos_tokens": 145, + "val/ratio": 0.9906548261642456, + "val/ratio_var": 8.349393465323374e-05 + }, + { + "episode": 9104, + "epoch": 0.4597979797979798, + "eps": 0, + "loss/policy_avg": 0.03359239920973778, + "loss/value_avg": 0.5841665863990784, + "lr": 6.0865444716359745e-06, + "objective/entropy": -75.15937042236328, + "objective/kl": 25.735273361206055, + "objective/non_score_reward": -1.2867636680603027, + "objective/rlhf_reward": 12.408548355102539, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009919216390699148, + "policy/clipfrac_avg": 0.00809397827833891, + "policy/entropy_avg": 0.15917348861694336, + "step": 569, + "val/clipfrac_avg": 0.11711117625236511, + "val/num_eos_tokens": 421, + "val/ratio": 0.9905821084976196, + "val/ratio_var": 5.5863049055915326e-05 + }, + { + "episode": 9120, + "epoch": 0.46060606060606063, + "eps": 0, + "loss/policy_avg": -0.1148596853017807, + "loss/value_avg": 0.9369633197784424, + "lr": 6.073502694879059e-06, + "objective/entropy": -101.9668197631836, + "objective/kl": 35.65166473388672, + "objective/non_score_reward": -1.782583236694336, + "objective/rlhf_reward": 13.006479263305664, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0008678468875586987, + "policy/clipfrac_avg": 0.009188210591673851, + "policy/entropy_avg": 0.1626424789428711, + "step": 570, + "val/clipfrac_avg": 0.04170583188533783, + "val/num_eos_tokens": 411, + "val/ratio": 0.9893851280212402, + "val/ratio_var": 6.942804611753672e-05 + }, + { + "episode": 9136, + "epoch": 0.4614141414141414, + "eps": 0, + "loss/policy_avg": 0.03529290482401848, + "loss/value_avg": 0.6290674209594727, + "lr": 6.060453257077686e-06, + "objective/entropy": -29.416015625, + "objective/kl": 34.878883361816406, + "objective/non_score_reward": -1.7439441680908203, + "objective/rlhf_reward": 13.38105583190918, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009020785801112652, + "policy/clipfrac_avg": 0.005981042981147766, + "policy/entropy_avg": 0.17604637145996094, + "step": 571, + "val/clipfrac_avg": 0.2255920171737671, + "val/num_eos_tokens": 197, + "val/ratio": 0.9879002571105957, + "val/ratio_var": 6.197366747073829e-05 + }, + { + "episode": 9152, + "epoch": 0.4622222222222222, + "eps": 0, + "loss/policy_avg": 0.07575584203004837, + "loss/value_avg": 0.48475825786590576, + "lr": 6.047396251359073e-06, + "objective/entropy": -94.31534576416016, + "objective/kl": 32.74857711791992, + "objective/non_score_reward": -1.6374289989471436, + "objective/rlhf_reward": 11.651633262634277, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.0009000588906928897, + "policy/clipfrac_avg": 0.006325645837932825, + "policy/entropy_avg": 0.16740036010742188, + "step": 572, + "val/clipfrac_avg": 0.1001669317483902, + "val/num_eos_tokens": 374, + "val/ratio": 0.9889869689941406, + "val/ratio_var": 4.305645416025072e-05 + }, + { + "episode": 9168, + "epoch": 0.463030303030303, + "eps": 0, + "loss/policy_avg": 0.06133081763982773, + "loss/value_avg": 0.6095678806304932, + "lr": 6.034331770904455e-06, + "objective/entropy": -134.7626495361328, + "objective/kl": 32.949615478515625, + "objective/non_score_reward": -1.6474809646606445, + "objective/rlhf_reward": 13.688455581665039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008799454662948847, + "policy/clipfrac_avg": 0.014949589036405087, + "policy/entropy_avg": 0.12122154235839844, + "step": 573, + "val/clipfrac_avg": 0.0686127319931984, + "val/num_eos_tokens": 434, + "val/ratio": 0.9921667575836182, + "val/ratio_var": 3.237271084799431e-05 + }, + { + "episode": 9184, + "epoch": 0.4638383838383838, + "eps": 0, + "loss/policy_avg": 0.0019814386032521725, + "loss/value_avg": 0.4525398015975952, + "lr": 6.0212599089484026e-06, + "objective/entropy": -40.367774963378906, + "objective/kl": 35.466163635253906, + "objective/non_score_reward": -1.773308277130127, + "objective/rlhf_reward": 11.523567199707031, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.001205465872772038, + "policy/clipfrac_avg": 0.008106344379484653, + "policy/entropy_avg": 0.2171649932861328, + "step": 574, + "val/clipfrac_avg": 0.13928186893463135, + "val/num_eos_tokens": 19, + "val/ratio": 0.9869014024734497, + "val/ratio_var": 5.974850137135945e-05 + }, + { + "episode": 9200, + "epoch": 0.46464646464646464, + "eps": 0, + "loss/policy_avg": -0.14026030898094177, + "loss/value_avg": 0.2715468406677246, + "lr": 6.008180758778167e-06, + "objective/entropy": -219.04417419433594, + "objective/kl": 33.518157958984375, + "objective/non_score_reward": -1.675907850265503, + "objective/rlhf_reward": 12.652216911315918, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009288907749578357, + "policy/clipfrac_avg": 0.014125748537480831, + "policy/entropy_avg": 0.12938570976257324, + "step": 575, + "val/clipfrac_avg": 0.09028111398220062, + "val/num_eos_tokens": 30, + "val/ratio": 0.9909526705741882, + "val/ratio_var": 6.470608786912635e-05 + }, + { + "episode": 9216, + "epoch": 0.46545454545454545, + "eps": 0, + "loss/policy_avg": 0.12112675607204437, + "loss/value_avg": 0.6767160892486572, + "lr": 5.9950944137330125e-06, + "objective/entropy": -93.06488037109375, + "objective/kl": 33.288169860839844, + "objective/non_score_reward": -1.6644084453582764, + "objective/rlhf_reward": 12.046529769897461, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009613010915927589, + "policy/clipfrac_avg": 0.006700174883008003, + "policy/entropy_avg": 0.17998027801513672, + "step": 576, + "val/clipfrac_avg": 0.12174967676401138, + "val/num_eos_tokens": 34, + "val/ratio": 0.988273561000824, + "val/ratio_var": 8.490013715345412e-05 + }, + { + "episode": 9232, + "epoch": 0.46626262626262627, + "eps": 0, + "loss/policy_avg": -0.041956715285778046, + "loss/value_avg": 0.5106937885284424, + "lr": 5.982000967203548e-06, + "objective/entropy": -71.49658966064453, + "objective/kl": 28.850629806518555, + "objective/non_score_reward": -1.4425315856933594, + "objective/rlhf_reward": 12.59653091430664, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009972681291401386, + "policy/clipfrac_avg": 0.009436940774321556, + "policy/entropy_avg": 0.1730337142944336, + "step": 577, + "val/clipfrac_avg": 0.1011686772108078, + "val/num_eos_tokens": 21, + "val/ratio": 0.9901687502861023, + "val/ratio_var": 5.90443960390985e-05 + }, + { + "episode": 9248, + "epoch": 0.4670707070707071, + "eps": 0, + "loss/policy_avg": -0.12406820058822632, + "loss/value_avg": 0.5695918798446655, + "lr": 5.968900512631065e-06, + "objective/entropy": -94.11358642578125, + "objective/kl": 33.98302459716797, + "objective/non_score_reward": -1.6991510391235352, + "objective/rlhf_reward": 13.886785507202148, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009757350198924541, + "policy/clipfrac_avg": 0.009374083951115608, + "policy/entropy_avg": 0.1509075164794922, + "step": 578, + "val/clipfrac_avg": 0.0964265689253807, + "val/num_eos_tokens": 261, + "val/ratio": 0.9889683723449707, + "val/ratio_var": 6.234552711248398e-05 + }, + { + "episode": 9264, + "epoch": 0.4678787878787879, + "eps": 0, + "loss/policy_avg": -0.012504605576395988, + "loss/value_avg": 0.16729792952537537, + "lr": 5.955793143506863e-06, + "objective/entropy": -60.78998565673828, + "objective/kl": 40.66157150268555, + "objective/non_score_reward": -2.033078670501709, + "objective/rlhf_reward": 13.380983352661133, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010061386274173856, + "policy/clipfrac_avg": 0.006983610801398754, + "policy/entropy_avg": 0.18235397338867188, + "step": 579, + "val/clipfrac_avg": 0.14943929016590118, + "val/num_eos_tokens": 86, + "val/ratio": 0.9873251914978027, + "val/ratio_var": 3.270859451731667e-05 + }, + { + "episode": 9280, + "epoch": 0.4686868686868687, + "eps": 0, + "loss/policy_avg": -0.05700581520795822, + "loss/value_avg": 0.6241277456283569, + "lr": 5.94267895337159e-06, + "objective/entropy": -80.41490173339844, + "objective/kl": 44.126407623291016, + "objective/non_score_reward": -2.206320285797119, + "objective/rlhf_reward": 11.426491737365723, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0009176756720989943, + "policy/clipfrac_avg": 0.005069363862276077, + "policy/entropy_avg": 0.17818450927734375, + "step": 580, + "val/clipfrac_avg": 0.18845900893211365, + "val/num_eos_tokens": 51, + "val/ratio": 0.9875844717025757, + "val/ratio_var": 7.17366929166019e-05 + }, + { + "episode": 9296, + "epoch": 0.4694949494949495, + "eps": 0, + "loss/policy_avg": -0.06188559532165527, + "loss/value_avg": 0.3465093970298767, + "lr": 5.929558035814574e-06, + "objective/entropy": -76.93428039550781, + "objective/kl": 37.18535614013672, + "objective/non_score_reward": -1.8592679500579834, + "objective/rlhf_reward": 12.875106811523438, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009798705577850342, + "policy/clipfrac_avg": 0.007772150449454784, + "policy/entropy_avg": 0.1757030487060547, + "step": 581, + "val/clipfrac_avg": 0.061571549624204636, + "val/num_eos_tokens": 158, + "val/ratio": 0.988136887550354, + "val/ratio_var": 4.800504757440649e-05 + }, + { + "episode": 9312, + "epoch": 0.4703030303030303, + "eps": 0, + "loss/policy_avg": 0.13406801223754883, + "loss/value_avg": 0.6718024611473083, + "lr": 5.916430484473149e-06, + "objective/entropy": -64.74710845947266, + "objective/kl": 39.926918029785156, + "objective/non_score_reward": -1.996346116065979, + "objective/rlhf_reward": 12.659903526306152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.000917055644094944, + "policy/clipfrac_avg": 0.007335194386541843, + "policy/entropy_avg": 0.18127059936523438, + "step": 582, + "val/clipfrac_avg": 0.1007988452911377, + "val/num_eos_tokens": 383, + "val/ratio": 0.9881880283355713, + "val/ratio_var": 5.9562738897511736e-05 + }, + { + "episode": 9328, + "epoch": 0.4711111111111111, + "eps": 0, + "loss/policy_avg": 0.3530195653438568, + "loss/value_avg": 0.4071064591407776, + "lr": 5.903296393031996e-06, + "objective/entropy": -120.16492462158203, + "objective/kl": 32.04585647583008, + "objective/non_score_reward": -1.602292776107788, + "objective/rlhf_reward": 13.272706985473633, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009271152084693313, + "policy/clipfrac_avg": 0.0157107375562191, + "policy/entropy_avg": 0.14319610595703125, + "step": 583, + "val/clipfrac_avg": 0.041299499571323395, + "val/num_eos_tokens": 319, + "val/ratio": 0.9893313646316528, + "val/ratio_var": 5.184482506592758e-05 + }, + { + "episode": 9344, + "epoch": 0.4719191919191919, + "eps": 0, + "loss/policy_avg": -0.03923230990767479, + "loss/value_avg": 0.5645366907119751, + "lr": 5.890155855222463e-06, + "objective/entropy": -21.60655975341797, + "objective/kl": 37.35072326660156, + "objective/non_score_reward": -1.8675363063812256, + "objective/rlhf_reward": 13.484025955200195, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001100124092772603, + "policy/clipfrac_avg": 0.010222227312624454, + "policy/entropy_avg": 0.17325496673583984, + "step": 584, + "val/clipfrac_avg": 0.09733504056930542, + "val/num_eos_tokens": 8, + "val/ratio": 0.9891010522842407, + "val/ratio_var": 5.214220436755568e-05 + }, + { + "episode": 9360, + "epoch": 0.4727272727272727, + "eps": 0, + "loss/policy_avg": -0.27854421734809875, + "loss/value_avg": 0.583124577999115, + "lr": 5.877008964821909e-06, + "objective/entropy": -169.64785766601562, + "objective/kl": 43.35786437988281, + "objective/non_score_reward": -2.167893171310425, + "objective/rlhf_reward": 12.074295043945312, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007353628752753139, + "policy/clipfrac_avg": 0.0040179151110351086, + "policy/entropy_avg": 0.13879728317260742, + "step": 585, + "val/clipfrac_avg": 0.21491241455078125, + "val/num_eos_tokens": 92, + "val/ratio": 0.990776777267456, + "val/ratio_var": 4.720980359707028e-05 + }, + { + "episode": 9376, + "epoch": 0.47353535353535353, + "eps": 0, + "loss/policy_avg": 0.051072392612695694, + "loss/value_avg": 0.36724406480789185, + "lr": 5.863855815653022e-06, + "objective/entropy": -151.75347900390625, + "objective/kl": 49.1492919921875, + "objective/non_score_reward": -2.4574646949768066, + "objective/rlhf_reward": 11.222222328186035, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0009073680848814547, + "policy/clipfrac_avg": 0.00646246736869216, + "policy/entropy_avg": 0.1743144989013672, + "step": 586, + "val/clipfrac_avg": 0.09604115784168243, + "val/num_eos_tokens": 162, + "val/ratio": 0.9888123273849487, + "val/ratio_var": 3.67878055840265e-05 + }, + { + "episode": 9392, + "epoch": 0.47434343434343434, + "eps": 0, + "loss/policy_avg": -0.07891325652599335, + "loss/value_avg": 0.20245110988616943, + "lr": 5.850696501583164e-06, + "objective/entropy": -66.69998931884766, + "objective/kl": 36.873565673828125, + "objective/non_score_reward": -1.8436782360076904, + "objective/rlhf_reward": 12.39850902557373, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0008563040173612535, + "policy/clipfrac_avg": 0.014813702553510666, + "policy/entropy_avg": 0.1890716552734375, + "step": 587, + "val/clipfrac_avg": 0.04891027510166168, + "val/num_eos_tokens": 361, + "val/ratio": 0.9900521039962769, + "val/ratio_var": 4.2384406697237864e-05 + }, + { + "episode": 9408, + "epoch": 0.47515151515151516, + "eps": 0, + "loss/policy_avg": 0.05903597176074982, + "loss/value_avg": 0.44623643159866333, + "lr": 5.837531116523683e-06, + "objective/entropy": -44.46873092651367, + "objective/kl": 37.953643798828125, + "objective/non_score_reward": -1.8976821899414062, + "objective/rlhf_reward": 13.446067810058594, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010375550482422113, + "policy/clipfrac_avg": 0.006371880881488323, + "policy/entropy_avg": 0.19812297821044922, + "step": 588, + "val/clipfrac_avg": 0.1483321189880371, + "val/num_eos_tokens": 409, + "val/ratio": 0.9867931604385376, + "val/ratio_var": 4.73033323942218e-05 + }, + { + "episode": 9424, + "epoch": 0.47595959595959597, + "eps": 0, + "loss/policy_avg": 0.10919662564992905, + "loss/value_avg": 0.3661342263221741, + "lr": 5.8243597544292585e-06, + "objective/entropy": -42.065185546875, + "objective/kl": 38.770042419433594, + "objective/non_score_reward": -1.9385021924972534, + "objective/rlhf_reward": 12.967748641967773, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011438718065619469, + "policy/clipfrac_avg": 0.010345622897148132, + "policy/entropy_avg": 0.20827293395996094, + "step": 589, + "val/clipfrac_avg": 0.05800151452422142, + "val/num_eos_tokens": 131, + "val/ratio": 0.9861246347427368, + "val/ratio_var": 5.3233892685966566e-05 + }, + { + "episode": 9440, + "epoch": 0.4767676767676768, + "eps": 0, + "loss/policy_avg": -0.10685920715332031, + "loss/value_avg": 0.6044321656227112, + "lr": 5.811182509297224e-06, + "objective/entropy": -48.181251525878906, + "objective/kl": 43.380714416503906, + "objective/non_score_reward": -2.1690359115600586, + "objective/rlhf_reward": 15.260650634765625, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0009571064729243517, + "policy/clipfrac_avg": 0.007153085432946682, + "policy/entropy_avg": 0.1891775131225586, + "step": 590, + "val/clipfrac_avg": 0.18321609497070312, + "val/num_eos_tokens": 322, + "val/ratio": 0.9876303672790527, + "val/ratio_var": 7.049312989693135e-05 + }, + { + "episode": 9456, + "epoch": 0.4775757575757576, + "eps": 0, + "loss/policy_avg": -0.14776897430419922, + "loss/value_avg": 0.4778248965740204, + "lr": 5.797999475166897e-06, + "objective/entropy": -93.8370361328125, + "objective/kl": 38.937896728515625, + "objective/non_score_reward": -1.946894884109497, + "objective/rlhf_reward": 13.224980354309082, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009119432652369142, + "policy/clipfrac_avg": 0.010985877364873886, + "policy/entropy_avg": 0.17043495178222656, + "step": 591, + "val/clipfrac_avg": 0.0706997662782669, + "val/num_eos_tokens": 209, + "val/ratio": 0.9888738989830017, + "val/ratio_var": 5.0667145842453465e-05 + }, + { + "episode": 9472, + "epoch": 0.4783838383838384, + "eps": 0, + "loss/policy_avg": 0.14408515393733978, + "loss/value_avg": 0.5567938089370728, + "lr": 5.784810746118907e-06, + "objective/entropy": -33.68360900878906, + "objective/kl": 35.89628601074219, + "objective/non_score_reward": -1.794814109802246, + "objective/rlhf_reward": 12.595810890197754, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008912064367905259, + "policy/clipfrac_avg": 0.011523826979100704, + "policy/entropy_avg": 0.13869476318359375, + "step": 592, + "val/clipfrac_avg": 0.15322092175483704, + "val/num_eos_tokens": 79, + "val/ratio": 0.9900076985359192, + "val/ratio_var": 2.749385566858109e-05 + }, + { + "episode": 9488, + "epoch": 0.47919191919191917, + "eps": 0, + "loss/policy_avg": 0.06612087786197662, + "loss/value_avg": 0.5821722149848938, + "lr": 5.77161641627453e-06, + "objective/entropy": -79.10285949707031, + "objective/kl": 34.06424331665039, + "objective/non_score_reward": -1.7032122611999512, + "objective/rlhf_reward": 12.25772476196289, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009309371234849095, + "policy/clipfrac_avg": 0.009452100843191147, + "policy/entropy_avg": 0.1573638916015625, + "step": 593, + "val/clipfrac_avg": 0.1111634224653244, + "val/num_eos_tokens": 63, + "val/ratio": 0.9895343780517578, + "val/ratio_var": 4.007742609246634e-05 + }, + { + "episode": 9504, + "epoch": 0.48, + "eps": 0, + "loss/policy_avg": -0.0943286195397377, + "loss/value_avg": 0.2797660827636719, + "lr": 5.7584165797950055e-06, + "objective/entropy": -173.3606719970703, + "objective/kl": 44.3161506652832, + "objective/non_score_reward": -2.2158076763153076, + "objective/rlhf_reward": 11.315442085266113, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0008013836340978742, + "policy/clipfrac_avg": 0.009280474856495857, + "policy/entropy_avg": 0.15043258666992188, + "step": 594, + "val/clipfrac_avg": 0.04634127765893936, + "val/num_eos_tokens": 439, + "val/ratio": 0.9904316067695618, + "val/ratio_var": 5.512699135579169e-05 + }, + { + "episode": 9520, + "epoch": 0.4808080808080808, + "eps": 0, + "loss/policy_avg": -0.17459821701049805, + "loss/value_avg": 0.568943977355957, + "lr": 5.745211330880872e-06, + "objective/entropy": -66.67430877685547, + "objective/kl": 39.01966094970703, + "objective/non_score_reward": -1.9509832859039307, + "objective/rlhf_reward": 14.345891952514648, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009961356408894062, + "policy/clipfrac_avg": 0.006623497232794762, + "policy/entropy_avg": 0.17397308349609375, + "step": 595, + "val/clipfrac_avg": 0.09633146226406097, + "val/num_eos_tokens": 305, + "val/ratio": 0.988785982131958, + "val/ratio_var": 4.24717363785021e-05 + }, + { + "episode": 9536, + "epoch": 0.4816161616161616, + "eps": 0, + "loss/policy_avg": -0.06750007718801498, + "loss/value_avg": 0.25791820883750916, + "lr": 5.732000763771301e-06, + "objective/entropy": -38.30708312988281, + "objective/kl": 36.13384246826172, + "objective/non_score_reward": -1.8066922426223755, + "objective/rlhf_reward": 14.466745376586914, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000953501439653337, + "policy/clipfrac_avg": 0.005453229416161776, + "policy/entropy_avg": 0.17740631103515625, + "step": 596, + "val/clipfrac_avg": 0.08646031469106674, + "val/num_eos_tokens": 232, + "val/ratio": 0.9876688718795776, + "val/ratio_var": 3.322769043734297e-05 + }, + { + "episode": 9552, + "epoch": 0.4824242424242424, + "eps": 0, + "loss/policy_avg": 0.0824112594127655, + "loss/value_avg": 1.29659903049469, + "lr": 5.71878497274341e-06, + "objective/entropy": -108.50715637207031, + "objective/kl": 25.864574432373047, + "objective/non_score_reward": -1.2932287454605103, + "objective/rlhf_reward": 12.941146850585938, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007267807959578931, + "policy/clipfrac_avg": 0.024973273277282715, + "policy/entropy_avg": 0.12932777404785156, + "step": 597, + "val/clipfrac_avg": 0.035626426339149475, + "val/num_eos_tokens": 90, + "val/ratio": 0.9911180734634399, + "val/ratio_var": 5.246651198831387e-05 + }, + { + "episode": 9568, + "epoch": 0.48323232323232324, + "eps": 0, + "loss/policy_avg": -0.036184508353471756, + "loss/value_avg": 0.2775241732597351, + "lr": 5.705564052111597e-06, + "objective/entropy": -73.0534896850586, + "objective/kl": 42.281578063964844, + "objective/non_score_reward": -2.114078998565674, + "objective/rlhf_reward": 14.260921478271484, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0010907119140028954, + "policy/clipfrac_avg": 0.007879609242081642, + "policy/entropy_avg": 0.18637657165527344, + "step": 598, + "val/clipfrac_avg": 0.0538523867726326, + "val/num_eos_tokens": 278, + "val/ratio": 0.9873398542404175, + "val/ratio_var": 5.951778803137131e-05 + }, + { + "episode": 9584, + "epoch": 0.48404040404040405, + "eps": 0, + "loss/policy_avg": -0.09331340342760086, + "loss/value_avg": 0.4313856363296509, + "lr": 5.69233809622687e-06, + "objective/entropy": -93.86756896972656, + "objective/kl": 36.109771728515625, + "objective/non_score_reward": -1.8054884672164917, + "objective/rlhf_reward": 12.616386413574219, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0010626658331602812, + "policy/clipfrac_avg": 0.006483650300651789, + "policy/entropy_avg": 0.15071487426757812, + "step": 599, + "val/clipfrac_avg": 0.053107958287000656, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895498752593994, + "val/ratio_var": 5.324727317201905e-05 + }, + { + "episode": 9600, + "epoch": 0.48484848484848486, + "eps": 0, + "loss/policy_avg": -0.1564587652683258, + "loss/value_avg": 0.3211984634399414, + "lr": 5.679107199476174e-06, + "objective/entropy": -186.87242126464844, + "objective/kl": 45.25263977050781, + "objective/non_score_reward": -2.262631893157959, + "objective/rlhf_reward": 13.534242630004883, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007487519760616124, + "policy/clipfrac_avg": 0.002827760763466358, + "policy/entropy_avg": 0.1533966064453125, + "step": 600, + "val/clipfrac_avg": 0.08722794055938721, + "val/num_eos_tokens": 503, + "val/ratio": 0.9899605512619019, + "val/ratio_var": 3.863383972202428e-05 + }, + { + "episode": 9616, + "epoch": 0.4856565656565657, + "eps": 0, + "loss/policy_avg": 0.2646865248680115, + "loss/value_avg": 0.9588360786437988, + "lr": 5.665871456281706e-06, + "objective/entropy": -43.572662353515625, + "objective/kl": 41.667076110839844, + "objective/non_score_reward": -2.0833539962768555, + "objective/rlhf_reward": 12.338520050048828, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012729446170851588, + "policy/clipfrac_avg": 0.015199655666947365, + "policy/entropy_avg": 0.1835641860961914, + "step": 601, + "val/clipfrac_avg": 0.20558898150920868, + "val/num_eos_tokens": 18, + "val/ratio": 0.9891852140426636, + "val/ratio_var": 8.858136425260454e-05 + }, + { + "episode": 9632, + "epoch": 0.4864646464646465, + "eps": 0, + "loss/policy_avg": 0.018315471708774567, + "loss/value_avg": 0.5322003364562988, + "lr": 5.65263096110026e-06, + "objective/entropy": -80.89665985107422, + "objective/kl": 40.64360809326172, + "objective/non_score_reward": -2.0321803092956543, + "objective/rlhf_reward": 13.131881713867188, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.000914716743864119, + "policy/clipfrac_avg": 0.0072946203872561455, + "policy/entropy_avg": 0.17550897598266602, + "step": 602, + "val/clipfrac_avg": 0.08266787230968475, + "val/num_eos_tokens": 181, + "val/ratio": 0.9880762696266174, + "val/ratio_var": 7.163242844399065e-05 + }, + { + "episode": 9648, + "epoch": 0.48727272727272725, + "eps": 0, + "loss/policy_avg": -0.22014892101287842, + "loss/value_avg": 0.27795735001564026, + "lr": 5.6393858084225305e-06, + "objective/entropy": -56.91044616699219, + "objective/kl": 37.012176513671875, + "objective/non_score_reward": -1.8506088256835938, + "objective/rlhf_reward": 13.821266174316406, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008462531259283423, + "policy/clipfrac_avg": 0.003934571985155344, + "policy/entropy_avg": 0.15899276733398438, + "step": 603, + "val/clipfrac_avg": 0.07405910640954971, + "val/num_eos_tokens": 119, + "val/ratio": 0.9893808960914612, + "val/ratio_var": 5.0625403673620895e-05 + }, + { + "episode": 9664, + "epoch": 0.48808080808080806, + "eps": 0, + "loss/policy_avg": -0.13129989802837372, + "loss/value_avg": 0.290170818567276, + "lr": 5.626136092772463e-06, + "objective/entropy": 0.5770502090454102, + "objective/kl": 45.011924743652344, + "objective/non_score_reward": -2.250596046447754, + "objective/rlhf_reward": 14.39002799987793, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011445335112512112, + "policy/clipfrac_avg": 0.004843855742365122, + "policy/entropy_avg": 0.21940231323242188, + "step": 604, + "val/clipfrac_avg": 0.053368281573057175, + "val/num_eos_tokens": 82, + "val/ratio": 0.9853905439376831, + "val/ratio_var": 4.1561663238098845e-05 + }, + { + "episode": 9680, + "epoch": 0.4888888888888889, + "eps": 0, + "loss/policy_avg": -0.15523457527160645, + "loss/value_avg": 1.380023717880249, + "lr": 5.612881908706555e-06, + "objective/entropy": -104.7314682006836, + "objective/kl": 29.19556999206543, + "objective/non_score_reward": -1.4597785472869873, + "objective/rlhf_reward": 13.54022216796875, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007493078010156751, + "policy/clipfrac_avg": 0.005996751599013805, + "policy/entropy_avg": 0.14516448974609375, + "step": 605, + "val/clipfrac_avg": 0.11536569148302078, + "val/num_eos_tokens": 465, + "val/ratio": 0.9902224540710449, + "val/ratio_var": 5.3489617130253464e-05 + }, + { + "episode": 9696, + "epoch": 0.4896969696969697, + "eps": 0, + "loss/policy_avg": -0.27331143617630005, + "loss/value_avg": 1.16130530834198, + "lr": 5.599623350813202e-06, + "objective/entropy": -157.1469268798828, + "objective/kl": 39.36458969116211, + "objective/non_score_reward": -1.9682295322418213, + "objective/rlhf_reward": 12.363801956176758, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008582321461290121, + "policy/clipfrac_avg": 0.010594002902507782, + "policy/entropy_avg": 0.14953947067260742, + "step": 606, + "val/clipfrac_avg": 0.0660855621099472, + "val/num_eos_tokens": 412, + "val/ratio": 0.9899133443832397, + "val/ratio_var": 4.759729199577123e-05 + }, + { + "episode": 9712, + "epoch": 0.4905050505050505, + "eps": 0, + "loss/policy_avg": -0.19870899617671967, + "loss/value_avg": 0.37283676862716675, + "lr": 5.586360513712011e-06, + "objective/entropy": -105.65791320800781, + "objective/kl": 33.51747131347656, + "objective/non_score_reward": -1.6758736371994019, + "objective/rlhf_reward": 13.281157493591309, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008841350791044533, + "policy/clipfrac_avg": 0.0069657666608691216, + "policy/entropy_avg": 0.15077972412109375, + "step": 607, + "val/clipfrac_avg": 0.0555139034986496, + "val/num_eos_tokens": 146, + "val/ratio": 0.9894367456436157, + "val/ratio_var": 7.082260708557442e-05 + }, + { + "episode": 9728, + "epoch": 0.4913131313131313, + "eps": 0, + "loss/policy_avg": 0.1218261867761612, + "loss/value_avg": 0.6498777866363525, + "lr": 5.57309349205312e-06, + "objective/entropy": -30.997875213623047, + "objective/kl": 36.81121063232422, + "objective/non_score_reward": -1.8405605554580688, + "objective/rlhf_reward": 12.971939086914062, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010020849294960499, + "policy/clipfrac_avg": 0.009872104972600937, + "policy/entropy_avg": 0.1610260009765625, + "step": 608, + "val/clipfrac_avg": 0.09282970428466797, + "val/num_eos_tokens": 288, + "val/ratio": 0.987944483757019, + "val/ratio_var": 3.414455568417907e-05 + }, + { + "episode": 9744, + "epoch": 0.4921212121212121, + "eps": 0, + "loss/policy_avg": -0.11768662929534912, + "loss/value_avg": 0.7992165684700012, + "lr": 5.559822380516539e-06, + "objective/entropy": -79.7080078125, + "objective/kl": 36.83974838256836, + "objective/non_score_reward": -1.8419876098632812, + "objective/rlhf_reward": 11.478324890136719, + "objective/scores": 13.3125, + "policy/approxkl_avg": 0.000889924238435924, + "policy/clipfrac_avg": 0.012620176188647747, + "policy/entropy_avg": 0.1525745391845703, + "step": 609, + "val/clipfrac_avg": 0.05175953358411789, + "val/num_eos_tokens": 13, + "val/ratio": 0.9897374510765076, + "val/ratio_var": 4.77236790175084e-05 + }, + { + "episode": 9760, + "epoch": 0.49292929292929294, + "eps": 0, + "loss/policy_avg": -0.2454073429107666, + "loss/value_avg": 1.018176555633545, + "lr": 5.546547273811465e-06, + "objective/entropy": -32.14586639404297, + "objective/kl": 33.13558578491211, + "objective/non_score_reward": -1.656779408454895, + "objective/rlhf_reward": 12.784626007080078, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0012670141877606511, + "policy/clipfrac_avg": 0.00890309177339077, + "policy/entropy_avg": 0.1815023422241211, + "step": 610, + "val/clipfrac_avg": 0.12171462923288345, + "val/num_eos_tokens": 215, + "val/ratio": 0.986750602722168, + "val/ratio_var": 5.6634431530255824e-05 + }, + { + "episode": 9776, + "epoch": 0.49373737373737375, + "eps": 0, + "loss/policy_avg": -0.05234597623348236, + "loss/value_avg": 0.9506372213363647, + "lr": 5.533268266675601e-06, + "objective/entropy": -21.851404190063477, + "objective/kl": 32.43745803833008, + "objective/non_score_reward": -1.6218730211257935, + "objective/rlhf_reward": 15.729690551757812, + "objective/scores": 17.375, + "policy/approxkl_avg": 0.0015964140184223652, + "policy/clipfrac_avg": 0.010030158795416355, + "policy/entropy_avg": 0.1926116943359375, + "step": 611, + "val/clipfrac_avg": 0.19708402454853058, + "val/num_eos_tokens": 8, + "val/ratio": 0.9859472513198853, + "val/ratio_var": 9.207271068589762e-05 + }, + { + "episode": 9792, + "epoch": 0.49454545454545457, + "eps": 0, + "loss/policy_avg": -0.0021607354283332825, + "loss/value_avg": 0.45648711919784546, + "lr": 5.5199854538744905e-06, + "objective/entropy": -77.98741149902344, + "objective/kl": 35.33883285522461, + "objective/non_score_reward": -1.7669416666030884, + "objective/rlhf_reward": 14.93618392944336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007928519044071436, + "policy/clipfrac_avg": 0.029583897441625595, + "policy/entropy_avg": 0.1448807716369629, + "step": 612, + "val/clipfrac_avg": 0.07284848392009735, + "val/num_eos_tokens": 735, + "val/ratio": 0.9906433820724487, + "val/ratio_var": 5.276045703794807e-05 + }, + { + "episode": 9808, + "epoch": 0.4953535353535354, + "eps": 0, + "loss/policy_avg": -0.08518673479557037, + "loss/value_avg": 0.6295963525772095, + "lr": 5.50669893020083e-06, + "objective/entropy": -163.5639190673828, + "objective/kl": 38.19306182861328, + "objective/non_score_reward": -1.9096529483795166, + "objective/rlhf_reward": 11.535659790039062, + "objective/scores": 13.4375, + "policy/approxkl_avg": 0.000834503909572959, + "policy/clipfrac_avg": 0.006998932920396328, + "policy/entropy_avg": 0.2186279296875, + "step": 613, + "val/clipfrac_avg": 0.0412946417927742, + "val/num_eos_tokens": 393, + "val/ratio": 0.9895168542861938, + "val/ratio_var": 4.984688348486088e-05 + }, + { + "episode": 9824, + "epoch": 0.49616161616161614, + "eps": 0, + "loss/policy_avg": -0.09563255310058594, + "loss/value_avg": 0.3395100235939026, + "lr": 5.493408790473808e-06, + "objective/entropy": -78.88749694824219, + "objective/kl": 36.03221893310547, + "objective/non_score_reward": -1.8016109466552734, + "objective/rlhf_reward": 13.745264053344727, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0010543245589360595, + "policy/clipfrac_avg": 0.0122576504945755, + "policy/entropy_avg": 0.17931032180786133, + "step": 614, + "val/clipfrac_avg": 0.03431976959109306, + "val/num_eos_tokens": 348, + "val/ratio": 0.9878480434417725, + "val/ratio_var": 7.280903082573786e-05 + }, + { + "episode": 9840, + "epoch": 0.49696969696969695, + "eps": 0, + "loss/policy_avg": -0.011238239705562592, + "loss/value_avg": 1.1844186782836914, + "lr": 5.480115129538409e-06, + "objective/entropy": -71.89592742919922, + "objective/kl": 29.166854858398438, + "objective/non_score_reward": -1.4583427906036377, + "objective/rlhf_reward": 11.682282447814941, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.0010009764228016138, + "policy/clipfrac_avg": 0.009198974817991257, + "policy/entropy_avg": 0.1814279556274414, + "step": 615, + "val/clipfrac_avg": 0.15483875572681427, + "val/num_eos_tokens": 405, + "val/ratio": 0.9910566806793213, + "val/ratio_var": 0.00011891258327523246 + }, + { + "episode": 9856, + "epoch": 0.49777777777777776, + "eps": 0, + "loss/policy_avg": 0.07725506275892258, + "loss/value_avg": 1.2075074911117554, + "lr": 5.466818042264754e-06, + "objective/entropy": -75.9026870727539, + "objective/kl": 39.84773635864258, + "objective/non_score_reward": -1.992386817932129, + "objective/rlhf_reward": 10.148238182067871, + "objective/scores": 12.125, + "policy/approxkl_avg": 0.0012564335484057665, + "policy/clipfrac_avg": 0.009645252488553524, + "policy/entropy_avg": 0.19198989868164062, + "step": 616, + "val/clipfrac_avg": 0.09601636976003647, + "val/num_eos_tokens": 432, + "val/ratio": 0.9873752593994141, + "val/ratio_var": 8.283311763079837e-05 + }, + { + "episode": 9872, + "epoch": 0.4985858585858586, + "eps": 0, + "loss/policy_avg": -0.12253916263580322, + "loss/value_avg": 0.34518784284591675, + "lr": 5.453517623547412e-06, + "objective/entropy": -117.6833267211914, + "objective/kl": 39.151649475097656, + "objective/non_score_reward": -1.9575825929641724, + "objective/rlhf_reward": 13.112730026245117, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008085196604952216, + "policy/clipfrac_avg": 0.01115940697491169, + "policy/entropy_avg": 0.16089963912963867, + "step": 617, + "val/clipfrac_avg": 0.08139624446630478, + "val/num_eos_tokens": 740, + "val/ratio": 0.9896994829177856, + "val/ratio_var": 3.962041228078306e-05 + }, + { + "episode": 9888, + "epoch": 0.4993939393939394, + "eps": 0, + "loss/policy_avg": 0.13594239950180054, + "loss/value_avg": 0.6978110074996948, + "lr": 5.440213968304728e-06, + "objective/entropy": -76.56217193603516, + "objective/kl": 27.95919418334961, + "objective/non_score_reward": -1.3979597091674805, + "objective/rlhf_reward": 15.203603744506836, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008653640979900956, + "policy/clipfrac_avg": 0.009502979926764965, + "policy/entropy_avg": 0.16449260711669922, + "step": 618, + "val/clipfrac_avg": 0.03875832259654999, + "val/num_eos_tokens": 5, + "val/ratio": 0.9884722828865051, + "val/ratio_var": 7.256936805788428e-05 + }, + { + "episode": 9904, + "epoch": 0.5002020202020202, + "eps": 0, + "loss/policy_avg": 0.03163297474384308, + "loss/value_avg": 0.5913329124450684, + "lr": 5.426907171478143e-06, + "objective/entropy": -51.29543685913086, + "objective/kl": 35.96702194213867, + "objective/non_score_reward": -1.7983510494232178, + "objective/rlhf_reward": 13.365711212158203, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010904096998274326, + "policy/clipfrac_avg": 0.007610537111759186, + "policy/entropy_avg": 0.18178224563598633, + "step": 619, + "val/clipfrac_avg": 0.1835818588733673, + "val/num_eos_tokens": 139, + "val/ratio": 0.9881883859634399, + "val/ratio_var": 4.50633124273736e-05 + }, + { + "episode": 9920, + "epoch": 0.501010101010101, + "eps": 0, + "loss/policy_avg": -0.018618369475007057, + "loss/value_avg": 0.1497330665588379, + "lr": 5.413597328031522e-06, + "objective/entropy": -40.40581130981445, + "objective/kl": 41.84601593017578, + "objective/non_score_reward": -2.0923008918762207, + "objective/rlhf_reward": 14.431137084960938, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009933523833751678, + "policy/clipfrac_avg": 0.004145262762904167, + "policy/entropy_avg": 0.1963958740234375, + "step": 620, + "val/clipfrac_avg": 0.017992915585637093, + "val/num_eos_tokens": 503, + "val/ratio": 0.9876341819763184, + "val/ratio_var": 2.160732947231736e-05 + }, + { + "episode": 9936, + "epoch": 0.5018181818181818, + "eps": 0, + "loss/policy_avg": 0.0749528706073761, + "loss/value_avg": 0.6211892366409302, + "lr": 5.4002845329504675e-06, + "objective/entropy": -250.99803161621094, + "objective/kl": 34.00743103027344, + "objective/non_score_reward": -1.7003716230392456, + "objective/rlhf_reward": 13.354316711425781, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008134699310176075, + "policy/clipfrac_avg": 0.019926324486732483, + "policy/entropy_avg": 0.15630340576171875, + "step": 621, + "val/clipfrac_avg": 0.060360997915267944, + "val/num_eos_tokens": 21, + "val/ratio": 0.9913326501846313, + "val/ratio_var": 4.280313805793412e-05 + }, + { + "episode": 9952, + "epoch": 0.5026262626262626, + "eps": 0, + "loss/policy_avg": 0.36562579870224, + "loss/value_avg": 1.0110520124435425, + "lr": 5.386968881241649e-06, + "objective/entropy": -123.3650894165039, + "objective/kl": 22.701366424560547, + "objective/non_score_reward": -1.1350682973861694, + "objective/rlhf_reward": 13.3727445602417, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009935309644788504, + "policy/clipfrac_avg": 0.008781377226114273, + "policy/entropy_avg": 0.1174173355102539, + "step": 622, + "val/clipfrac_avg": 0.14134380221366882, + "val/num_eos_tokens": 221, + "val/ratio": 0.9914640188217163, + "val/ratio_var": 6.538646994158626e-05 + }, + { + "episode": 9968, + "epoch": 0.5034343434343435, + "eps": 0, + "loss/policy_avg": 0.21662172675132751, + "loss/value_avg": 0.46216171979904175, + "lr": 5.373650467932122e-06, + "objective/entropy": -120.33831787109375, + "objective/kl": 36.585201263427734, + "objective/non_score_reward": -1.8292601108551025, + "objective/rlhf_reward": 12.944177627563477, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0015983078628778458, + "policy/clipfrac_avg": 0.027648624032735825, + "policy/entropy_avg": 0.19037723541259766, + "step": 623, + "val/clipfrac_avg": 0.1728610098361969, + "val/num_eos_tokens": 333, + "val/ratio": 0.9866871237754822, + "val/ratio_var": 0.00018466412439011037 + }, + { + "episode": 9984, + "epoch": 0.5042424242424243, + "eps": 0, + "loss/policy_avg": 0.03029388189315796, + "loss/value_avg": 0.5422893762588501, + "lr": 5.360329388068649e-06, + "objective/entropy": -107.19503784179688, + "objective/kl": 34.64921951293945, + "objective/non_score_reward": -1.7324607372283936, + "objective/rlhf_reward": 11.345664024353027, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0009691518498584628, + "policy/clipfrac_avg": 0.006136470474302769, + "policy/entropy_avg": 0.1900615692138672, + "step": 624, + "val/clipfrac_avg": 0.06680267304182053, + "val/num_eos_tokens": 650, + "val/ratio": 0.988233208656311, + "val/ratio_var": 7.25203353795223e-05 + }, + { + "episode": 10000, + "epoch": 0.5050505050505051, + "eps": 0, + "loss/policy_avg": 0.2465195208787918, + "loss/value_avg": 0.6872731447219849, + "lr": 5.347005736717024e-06, + "objective/entropy": -101.5595474243164, + "objective/kl": 37.40558624267578, + "objective/non_score_reward": -1.8702794313430786, + "objective/rlhf_reward": 13.192220687866211, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008968232432380319, + "policy/clipfrac_avg": 0.014711007475852966, + "policy/entropy_avg": 0.14348411560058594, + "step": 625, + "val/clipfrac_avg": 0.09929686039686203, + "val/num_eos_tokens": 428, + "val/ratio": 0.9888505935668945, + "val/ratio_var": 4.519468348007649e-05 + }, + { + "episode": 10016, + "epoch": 0.5058585858585859, + "eps": 0, + "loss/policy_avg": -0.09769874811172485, + "loss/value_avg": 0.2699797451496124, + "lr": 5.333679608961389e-06, + "objective/entropy": -147.00888061523438, + "objective/kl": 39.13426971435547, + "objective/non_score_reward": -1.9567137956619263, + "objective/rlhf_reward": 14.191723823547363, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008501397678628564, + "policy/clipfrac_avg": 0.00891679897904396, + "policy/entropy_avg": 0.19576263427734375, + "step": 626, + "val/clipfrac_avg": 0.0778140276670456, + "val/num_eos_tokens": 156, + "val/ratio": 0.9883639812469482, + "val/ratio_var": 7.028185063973069e-05 + }, + { + "episode": 10032, + "epoch": 0.5066666666666667, + "eps": 0, + "loss/policy_avg": -0.15752656757831573, + "loss/value_avg": 1.4141845703125, + "lr": 5.320351099903565e-06, + "objective/entropy": -60.07448196411133, + "objective/kl": 31.409622192382812, + "objective/non_score_reward": -1.5704810619354248, + "objective/rlhf_reward": 14.788894653320312, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000859480001963675, + "policy/clipfrac_avg": 0.007050718180835247, + "policy/entropy_avg": 0.1713123321533203, + "step": 627, + "val/clipfrac_avg": 0.11736910045146942, + "val/num_eos_tokens": 235, + "val/ratio": 0.9884116649627686, + "val/ratio_var": 6.11452414887026e-05 + }, + { + "episode": 10048, + "epoch": 0.5074747474747475, + "eps": 0, + "loss/policy_avg": -0.3966858983039856, + "loss/value_avg": 0.726300835609436, + "lr": 5.307020304662361e-06, + "objective/entropy": -94.50772094726562, + "objective/kl": 32.27813720703125, + "objective/non_score_reward": -1.6139068603515625, + "objective/rlhf_reward": 13.768905639648438, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0007259462727233768, + "policy/clipfrac_avg": 0.005907295271754265, + "policy/entropy_avg": 0.13324737548828125, + "step": 628, + "val/clipfrac_avg": 0.2497040331363678, + "val/num_eos_tokens": 195, + "val/ratio": 0.9917147159576416, + "val/ratio_var": 4.8239635361824185e-05 + }, + { + "episode": 10064, + "epoch": 0.5082828282828283, + "eps": 0, + "loss/policy_avg": 0.10847073793411255, + "loss/value_avg": 1.0482819080352783, + "lr": 5.293687318372907e-06, + "objective/entropy": -56.479736328125, + "objective/kl": 30.607051849365234, + "objective/non_score_reward": -1.5303525924682617, + "objective/rlhf_reward": 12.360271453857422, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0010972167365252972, + "policy/clipfrac_avg": 0.00503067672252655, + "policy/entropy_avg": 0.1625370979309082, + "step": 629, + "val/clipfrac_avg": 0.18077512085437775, + "val/num_eos_tokens": 22, + "val/ratio": 0.9892880916595459, + "val/ratio_var": 6.566623051185161e-05 + }, + { + "episode": 10080, + "epoch": 0.509090909090909, + "eps": 0, + "loss/policy_avg": -0.23807989060878754, + "loss/value_avg": 0.4602620005607605, + "lr": 5.2803522361859596e-06, + "objective/entropy": -135.9888916015625, + "objective/kl": 39.755985260009766, + "objective/non_score_reward": -1.9877994060516357, + "objective/rlhf_reward": 12.520012855529785, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009651440195739269, + "policy/clipfrac_avg": 0.008618216961622238, + "policy/entropy_avg": 0.17929983139038086, + "step": 630, + "val/clipfrac_avg": 0.08070136606693268, + "val/num_eos_tokens": 379, + "val/ratio": 0.9886597990989685, + "val/ratio_var": 4.691245703725144e-05 + }, + { + "episode": 10096, + "epoch": 0.5098989898989899, + "eps": 0, + "loss/policy_avg": 0.23663482069969177, + "loss/value_avg": 0.8216120004653931, + "lr": 5.267015153267246e-06, + "objective/entropy": -94.9400634765625, + "objective/kl": 37.73855209350586, + "objective/non_score_reward": -1.886927604675293, + "objective/rlhf_reward": 13.628697395324707, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009247865527868271, + "policy/clipfrac_avg": 0.009060590527951717, + "policy/entropy_avg": 0.16341590881347656, + "step": 631, + "val/clipfrac_avg": 0.14852876961231232, + "val/num_eos_tokens": 182, + "val/ratio": 0.988681435585022, + "val/ratio_var": 6.258696521399543e-05 + }, + { + "episode": 10112, + "epoch": 0.5107070707070707, + "eps": 0, + "loss/policy_avg": -0.08203352987766266, + "loss/value_avg": 0.8431352376937866, + "lr": 5.25367616479676e-06, + "objective/entropy": -80.35845947265625, + "objective/kl": 36.896602630615234, + "objective/non_score_reward": -1.8448302745819092, + "objective/rlhf_reward": 11.233295440673828, + "objective/scores": 13.0625, + "policy/approxkl_avg": 0.0012214542366564274, + "policy/clipfrac_avg": 0.006619518622756004, + "policy/entropy_avg": 0.19206619262695312, + "step": 632, + "val/clipfrac_avg": 0.08380433917045593, + "val/num_eos_tokens": 526, + "val/ratio": 0.9882425665855408, + "val/ratio_var": 0.00010104960529133677 + }, + { + "episode": 10128, + "epoch": 0.5115151515151515, + "eps": 0, + "loss/policy_avg": -0.20128124952316284, + "loss/value_avg": 0.36594098806381226, + "lr": 5.240335365968104e-06, + "objective/entropy": -83.84121704101562, + "objective/kl": 31.87903594970703, + "objective/non_score_reward": -1.59395170211792, + "objective/rlhf_reward": 11.906047821044922, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.00077263661660254, + "policy/clipfrac_avg": 0.008104969747364521, + "policy/entropy_avg": 0.1480419635772705, + "step": 633, + "val/clipfrac_avg": 0.10230521857738495, + "val/num_eos_tokens": 340, + "val/ratio": 0.9903359413146973, + "val/ratio_var": 5.447642979561351e-05 + }, + { + "episode": 10144, + "epoch": 0.5123232323232323, + "eps": 0, + "loss/policy_avg": 0.45129337906837463, + "loss/value_avg": 1.256578803062439, + "lr": 5.226992851987793e-06, + "objective/entropy": -79.21052551269531, + "objective/kl": 34.82407760620117, + "objective/non_score_reward": -1.741204023361206, + "objective/rlhf_reward": 12.868170738220215, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009523687185719609, + "policy/clipfrac_avg": 0.009445530362427235, + "policy/entropy_avg": 0.18542003631591797, + "step": 634, + "val/clipfrac_avg": 0.2403840720653534, + "val/num_eos_tokens": 511, + "val/ratio": 0.9878572821617126, + "val/ratio_var": 5.5977081501623616e-05 + }, + { + "episode": 10160, + "epoch": 0.5131313131313131, + "eps": 0, + "loss/policy_avg": -0.2209135740995407, + "loss/value_avg": 0.4546852111816406, + "lr": 5.213648718074584e-06, + "objective/entropy": -60.33174133300781, + "objective/kl": 41.36100769042969, + "objective/non_score_reward": -2.0680503845214844, + "objective/rlhf_reward": 13.408512115478516, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0012648601550608873, + "policy/clipfrac_avg": 0.013032745569944382, + "policy/entropy_avg": 0.1959362030029297, + "step": 635, + "val/clipfrac_avg": 0.0490388460457325, + "val/num_eos_tokens": 286, + "val/ratio": 0.9876245260238647, + "val/ratio_var": 5.1731625717366114e-05 + }, + { + "episode": 10176, + "epoch": 0.5139393939393939, + "eps": 0, + "loss/policy_avg": 0.30760645866394043, + "loss/value_avg": 1.3313541412353516, + "lr": 5.2003030594587964e-06, + "objective/entropy": -65.56263732910156, + "objective/kl": 30.6090030670166, + "objective/non_score_reward": -1.5304501056671143, + "objective/rlhf_reward": 12.985175132751465, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009048302308656275, + "policy/clipfrac_avg": 0.009688157588243484, + "policy/entropy_avg": 0.16722917556762695, + "step": 636, + "val/clipfrac_avg": 0.08400047570466995, + "val/num_eos_tokens": 173, + "val/ratio": 0.988949716091156, + "val/ratio_var": 5.8918587455991656e-05 + }, + { + "episode": 10192, + "epoch": 0.5147474747474747, + "eps": 0, + "loss/policy_avg": -0.03748584911227226, + "loss/value_avg": 0.3042546808719635, + "lr": 5.18695597138163e-06, + "objective/entropy": -102.1771240234375, + "objective/kl": 46.15885543823242, + "objective/non_score_reward": -2.3079428672790527, + "objective/rlhf_reward": 11.965494155883789, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009750047465786338, + "policy/clipfrac_avg": 0.008077681064605713, + "policy/entropy_avg": 0.168487548828125, + "step": 637, + "val/clipfrac_avg": 0.051075853407382965, + "val/num_eos_tokens": 307, + "val/ratio": 0.98898845911026, + "val/ratio_var": 4.223840733175166e-05 + }, + { + "episode": 10208, + "epoch": 0.5155555555555555, + "eps": 0, + "loss/policy_avg": -0.23826318979263306, + "loss/value_avg": 0.7353537082672119, + "lr": 5.1736075490944815e-06, + "objective/entropy": -136.01560974121094, + "objective/kl": 37.841087341308594, + "objective/non_score_reward": -1.8920543193817139, + "objective/rlhf_reward": 12.225133895874023, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.001157400431111455, + "policy/clipfrac_avg": 0.008751263841986656, + "policy/entropy_avg": 0.19327354431152344, + "step": 638, + "val/clipfrac_avg": 0.0865342766046524, + "val/num_eos_tokens": 1837, + "val/ratio": 0.9871768951416016, + "val/ratio_var": 6.5818581788335e-05 + }, + { + "episode": 10224, + "epoch": 0.5163636363636364, + "eps": 0, + "loss/policy_avg": 0.015793576836586, + "loss/value_avg": 0.3555375635623932, + "lr": 5.160257887858278e-06, + "objective/entropy": -36.917694091796875, + "objective/kl": 37.96650314331055, + "objective/non_score_reward": -1.8983252048492432, + "objective/rlhf_reward": 14.390737533569336, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.000980690703727305, + "policy/clipfrac_avg": 0.007268393412232399, + "policy/entropy_avg": 0.17595863342285156, + "step": 639, + "val/clipfrac_avg": 0.04352998733520508, + "val/num_eos_tokens": 116, + "val/ratio": 0.9883407354354858, + "val/ratio_var": 4.279622953617945e-05 + }, + { + "episode": 10240, + "epoch": 0.5171717171717172, + "eps": 0, + "loss/policy_avg": -0.15323098003864288, + "loss/value_avg": 0.3132088780403137, + "lr": 5.146907082942778e-06, + "objective/entropy": -150.77487182617188, + "objective/kl": 33.419410705566406, + "objective/non_score_reward": -1.6709705591201782, + "objective/rlhf_reward": 12.969654083251953, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007614089408889413, + "policy/clipfrac_avg": 0.00750840874388814, + "policy/entropy_avg": 0.14741188287734985, + "step": 640, + "val/clipfrac_avg": 0.06061185896396637, + "val/num_eos_tokens": 86, + "val/ratio": 0.9901533126831055, + "val/ratio_var": 4.773136606672779e-05 + }, + { + "episode": 10256, + "epoch": 0.517979797979798, + "eps": 0, + "loss/policy_avg": -0.40618008375167847, + "loss/value_avg": 0.39824044704437256, + "lr": 5.133555229625912e-06, + "objective/entropy": -184.5850067138672, + "objective/kl": 46.53107452392578, + "objective/non_score_reward": -2.3265538215637207, + "objective/rlhf_reward": 13.353133201599121, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008069286122918129, + "policy/clipfrac_avg": 0.004317488521337509, + "policy/entropy_avg": 0.15611839294433594, + "step": 641, + "val/clipfrac_avg": 0.20341260731220245, + "val/num_eos_tokens": 56, + "val/ratio": 0.9897021055221558, + "val/ratio_var": 4.78306865261402e-05 + }, + { + "episode": 10272, + "epoch": 0.5187878787878788, + "eps": 0, + "loss/policy_avg": 0.35164520144462585, + "loss/value_avg": 0.761013925075531, + "lr": 5.120202423193085e-06, + "objective/entropy": -92.79855346679688, + "objective/kl": 32.67811965942383, + "objective/non_score_reward": -1.6339060068130493, + "objective/rlhf_reward": 11.483281135559082, + "objective/scores": 13.125, + "policy/approxkl_avg": 0.000959779426921159, + "policy/clipfrac_avg": 0.013718648813664913, + "policy/entropy_avg": 0.2091999053955078, + "step": 642, + "val/clipfrac_avg": 0.10013719648122787, + "val/num_eos_tokens": 227, + "val/ratio": 0.9883617162704468, + "val/ratio_var": 7.446538074873388e-05 + }, + { + "episode": 10288, + "epoch": 0.5195959595959596, + "eps": 0, + "loss/policy_avg": 0.17767910659313202, + "loss/value_avg": 0.5378450155258179, + "lr": 5.106848758936508e-06, + "objective/entropy": -61.3033447265625, + "objective/kl": 27.626182556152344, + "objective/non_score_reward": -1.381309151649475, + "objective/rlhf_reward": 14.431190490722656, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000927220331504941, + "policy/clipfrac_avg": 0.01159143727272749, + "policy/entropy_avg": 0.17319107055664062, + "step": 643, + "val/clipfrac_avg": 0.08490148931741714, + "val/num_eos_tokens": 201, + "val/ratio": 0.9895293712615967, + "val/ratio_var": 5.0640650442801416e-05 + }, + { + "episode": 10304, + "epoch": 0.5204040404040404, + "eps": 0, + "loss/policy_avg": -0.004831582307815552, + "loss/value_avg": 0.30399805307388306, + "lr": 5.093494332154511e-06, + "objective/entropy": -100.7375717163086, + "objective/kl": 42.24378967285156, + "objective/non_score_reward": -2.112189292907715, + "objective/rlhf_reward": 12.333123207092285, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0008687187219038606, + "policy/clipfrac_avg": 0.005677542649209499, + "policy/entropy_avg": 0.17821407318115234, + "step": 644, + "val/clipfrac_avg": 0.03101455792784691, + "val/num_eos_tokens": 530, + "val/ratio": 0.9884386658668518, + "val/ratio_var": 4.8310637794202194e-05 + }, + { + "episode": 10320, + "epoch": 0.5212121212121212, + "eps": 0, + "loss/policy_avg": -0.1480829268693924, + "loss/value_avg": 1.2215086221694946, + "lr": 5.080139238150869e-06, + "objective/entropy": -77.07893371582031, + "objective/kl": 44.610504150390625, + "objective/non_score_reward": -2.230525255203247, + "objective/rlhf_reward": 10.777286529541016, + "objective/scores": 13.0, + "policy/approxkl_avg": 0.00098347757011652, + "policy/clipfrac_avg": 0.0057642534375190735, + "policy/entropy_avg": 0.20584464073181152, + "step": 645, + "val/clipfrac_avg": 0.06691773980855942, + "val/num_eos_tokens": 44, + "val/ratio": 0.987593412399292, + "val/ratio_var": 6.559812754858285e-05 + }, + { + "episode": 10336, + "epoch": 0.522020202020202, + "eps": 0, + "loss/policy_avg": 0.021136268973350525, + "loss/value_avg": 0.24227744340896606, + "lr": 5.066783572234115e-06, + "objective/entropy": -102.19164276123047, + "objective/kl": 37.040985107421875, + "objective/non_score_reward": -1.8520495891571045, + "objective/rlhf_reward": 15.030763626098633, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008576101390644908, + "policy/clipfrac_avg": 0.005198730155825615, + "policy/entropy_avg": 0.1648244857788086, + "step": 646, + "val/clipfrac_avg": 0.023988839238882065, + "val/num_eos_tokens": 250, + "val/ratio": 0.988852858543396, + "val/ratio_var": 3.1189680157694966e-05 + }, + { + "episode": 10352, + "epoch": 0.5228282828282829, + "eps": 0, + "loss/policy_avg": 0.1345120370388031, + "loss/value_avg": 0.6760163307189941, + "lr": 5.053427429716867e-06, + "objective/entropy": -93.60134887695312, + "objective/kl": 39.353092193603516, + "objective/non_score_reward": -1.9676547050476074, + "objective/rlhf_reward": 13.829219818115234, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0009072768152691424, + "policy/clipfrac_avg": 0.010927075520157814, + "policy/entropy_avg": 0.16292381286621094, + "step": 647, + "val/clipfrac_avg": 0.10246877372264862, + "val/num_eos_tokens": 159, + "val/ratio": 0.988797664642334, + "val/ratio_var": 6.221264629857615e-05 + }, + { + "episode": 10368, + "epoch": 0.5236363636363637, + "eps": 0, + "loss/policy_avg": -0.03434079885482788, + "loss/value_avg": 0.26276272535324097, + "lr": 5.040070905915139e-06, + "objective/entropy": -218.74851989746094, + "objective/kl": 45.733909606933594, + "objective/non_score_reward": -2.2866954803466797, + "objective/rlhf_reward": 13.22892951965332, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007575824274681509, + "policy/clipfrac_avg": 0.012265796773135662, + "policy/entropy_avg": 0.1510915756225586, + "step": 648, + "val/clipfrac_avg": 0.09911301732063293, + "val/num_eos_tokens": 287, + "val/ratio": 0.9897732138633728, + "val/ratio_var": 5.4691558034392074e-05 + }, + { + "episode": 10384, + "epoch": 0.5244444444444445, + "eps": 0, + "loss/policy_avg": 0.17616848647594452, + "loss/value_avg": 0.9873183965682983, + "lr": 5.026714096147673e-06, + "objective/entropy": -154.69427490234375, + "objective/kl": 38.24567794799805, + "objective/non_score_reward": -1.9122838973999023, + "objective/rlhf_reward": 12.079903602600098, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0007898483891040087, + "policy/clipfrac_avg": 0.004851070698350668, + "policy/entropy_avg": 0.18099498748779297, + "step": 649, + "val/clipfrac_avg": 0.05088340863585472, + "val/num_eos_tokens": 1593, + "val/ratio": 0.9899267554283142, + "val/ratio_var": 4.2158419091720134e-05 + }, + { + "episode": 10400, + "epoch": 0.5252525252525253, + "eps": 0, + "loss/policy_avg": -0.10312466323375702, + "loss/value_avg": 0.7348710298538208, + "lr": 5.013357095735246e-06, + "objective/entropy": -90.748779296875, + "objective/kl": 27.840486526489258, + "objective/non_score_reward": -1.392024278640747, + "objective/rlhf_reward": 14.357975006103516, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008530996856279671, + "policy/clipfrac_avg": 0.005986053496599197, + "policy/entropy_avg": 0.14681434631347656, + "step": 650, + "val/clipfrac_avg": 0.17565612494945526, + "val/num_eos_tokens": 591, + "val/ratio": 0.9898090362548828, + "val/ratio_var": 6.2022045312915e-05 + }, + { + "episode": 10416, + "epoch": 0.526060606060606, + "eps": 0, + "loss/policy_avg": -0.37787753343582153, + "loss/value_avg": 0.4072280526161194, + "lr": 5e-06, + "objective/entropy": -116.00996398925781, + "objective/kl": 36.00543975830078, + "objective/non_score_reward": -1.800271987915039, + "objective/rlhf_reward": 13.379415512084961, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008067564340308309, + "policy/clipfrac_avg": 0.006692061200737953, + "policy/entropy_avg": 0.15956878662109375, + "step": 651, + "val/clipfrac_avg": 0.12555305659770966, + "val/num_eos_tokens": 475, + "val/ratio": 0.9889513850212097, + "val/ratio_var": 7.936693145893514e-05 + }, + { + "episode": 10432, + "epoch": 0.5268686868686868, + "eps": 0, + "loss/policy_avg": -0.024874819442629814, + "loss/value_avg": 0.30301523208618164, + "lr": 4.986642904264754e-06, + "objective/entropy": -113.37627410888672, + "objective/kl": 35.3900260925293, + "objective/non_score_reward": -1.7695012092590332, + "objective/rlhf_reward": 12.878936767578125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009043759200721979, + "policy/clipfrac_avg": 0.011335442773997784, + "policy/entropy_avg": 0.16724395751953125, + "step": 652, + "val/clipfrac_avg": 0.05399475246667862, + "val/num_eos_tokens": 72, + "val/ratio": 0.989266037940979, + "val/ratio_var": 4.7810990508878604e-05 + }, + { + "episode": 10448, + "epoch": 0.5276767676767676, + "eps": 0, + "loss/policy_avg": -0.4576267898082733, + "loss/value_avg": 0.6310689449310303, + "lr": 4.973285903852329e-06, + "objective/entropy": -97.41447448730469, + "objective/kl": 31.19072914123535, + "objective/non_score_reward": -1.5595364570617676, + "objective/rlhf_reward": 14.838900566101074, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008020643144845963, + "policy/clipfrac_avg": 0.003582795150578022, + "policy/entropy_avg": 0.16522979736328125, + "step": 653, + "val/clipfrac_avg": 0.17964357137680054, + "val/num_eos_tokens": 25, + "val/ratio": 0.9892231225967407, + "val/ratio_var": 5.0224600272485986e-05 + }, + { + "episode": 10464, + "epoch": 0.5284848484848484, + "eps": 0, + "loss/policy_avg": -0.1491837352514267, + "loss/value_avg": 0.7328892946243286, + "lr": 4.959929094084862e-06, + "objective/entropy": -157.48797607421875, + "objective/kl": 33.45720672607422, + "objective/non_score_reward": -1.6728602647781372, + "objective/rlhf_reward": 12.967764854431152, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009867304470390081, + "policy/clipfrac_avg": 0.008453142829239368, + "policy/entropy_avg": 0.1548633575439453, + "step": 654, + "val/clipfrac_avg": 0.10647779703140259, + "val/num_eos_tokens": 1553, + "val/ratio": 0.98896324634552, + "val/ratio_var": 0.00010884697985602543 + }, + { + "episode": 10480, + "epoch": 0.5292929292929293, + "eps": 0, + "loss/policy_avg": -0.0017375200986862183, + "loss/value_avg": 0.4749241769313812, + "lr": 4.946572570283135e-06, + "objective/entropy": -151.52291870117188, + "objective/kl": 41.44767761230469, + "objective/non_score_reward": -2.0723838806152344, + "objective/rlhf_reward": 11.599491119384766, + "objective/scores": 13.6875, + "policy/approxkl_avg": 0.0010196359362453222, + "policy/clipfrac_avg": 0.008509906008839607, + "policy/entropy_avg": 0.17792892456054688, + "step": 655, + "val/clipfrac_avg": 0.13483279943466187, + "val/num_eos_tokens": 723, + "val/ratio": 0.9885885715484619, + "val/ratio_var": 3.869455758831464e-05 + }, + { + "episode": 10496, + "epoch": 0.5301010101010101, + "eps": 0, + "loss/policy_avg": 0.30413296818733215, + "loss/value_avg": 0.4696364402770996, + "lr": 4.933216427765886e-06, + "objective/entropy": -98.99279022216797, + "objective/kl": 37.215736389160156, + "objective/non_score_reward": -1.8607869148254395, + "objective/rlhf_reward": 12.459526062011719, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0008932605269365013, + "policy/clipfrac_avg": 0.01308157853782177, + "policy/entropy_avg": 0.16152191162109375, + "step": 656, + "val/clipfrac_avg": 0.09068673849105835, + "val/num_eos_tokens": 495, + "val/ratio": 0.9888198971748352, + "val/ratio_var": 6.496508285636082e-05 + }, + { + "episode": 10512, + "epoch": 0.5309090909090909, + "eps": 0, + "loss/policy_avg": 0.1066155657172203, + "loss/value_avg": 1.9890694618225098, + "lr": 4.919860761849132e-06, + "objective/entropy": -104.63330078125, + "objective/kl": 38.032981872558594, + "objective/non_score_reward": -1.9016492366790771, + "objective/rlhf_reward": 12.348350524902344, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007215352961793542, + "policy/clipfrac_avg": 0.003871732857078314, + "policy/entropy_avg": 0.1489715576171875, + "step": 657, + "val/clipfrac_avg": 0.17002993822097778, + "val/num_eos_tokens": 9, + "val/ratio": 0.9906018972396851, + "val/ratio_var": 3.7619625800289214e-05 + }, + { + "episode": 10528, + "epoch": 0.5317171717171717, + "eps": 0, + "loss/policy_avg": 0.04127347841858864, + "loss/value_avg": 0.2920118570327759, + "lr": 4.90650566784549e-06, + "objective/entropy": -83.36598205566406, + "objective/kl": 36.28617858886719, + "objective/non_score_reward": -1.8143088817596436, + "objective/rlhf_reward": 14.560690879821777, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008893824997358024, + "policy/clipfrac_avg": 0.01114499568939209, + "policy/entropy_avg": 0.16919898986816406, + "step": 658, + "val/clipfrac_avg": 0.032178279012441635, + "val/num_eos_tokens": 377, + "val/ratio": 0.9892511367797852, + "val/ratio_var": 3.130078039248474e-05 + }, + { + "episode": 10544, + "epoch": 0.5325252525252525, + "eps": 0, + "loss/policy_avg": 0.04267742112278938, + "loss/value_avg": 0.2801401913166046, + "lr": 4.893151241063493e-06, + "objective/entropy": -46.258995056152344, + "objective/kl": 32.11946105957031, + "objective/non_score_reward": -1.6059731245040894, + "objective/rlhf_reward": 12.776840209960938, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0012434273958206177, + "policy/clipfrac_avg": 0.008764926344156265, + "policy/entropy_avg": 0.1816234588623047, + "step": 659, + "val/clipfrac_avg": 0.07641617208719254, + "val/num_eos_tokens": 16, + "val/ratio": 0.9873616099357605, + "val/ratio_var": 0.0001170080722658895 + }, + { + "episode": 10560, + "epoch": 0.5333333333333333, + "eps": 0, + "loss/policy_avg": 0.20670434832572937, + "loss/value_avg": 0.484742134809494, + "lr": 4.879797576806915e-06, + "objective/entropy": -151.64088439941406, + "objective/kl": 39.09260559082031, + "objective/non_score_reward": -1.9546302556991577, + "objective/rlhf_reward": 11.217245101928711, + "objective/scores": 13.1875, + "policy/approxkl_avg": 0.0007958562346175313, + "policy/clipfrac_avg": 0.00742696039378643, + "policy/entropy_avg": 0.1595144271850586, + "step": 660, + "val/clipfrac_avg": 0.07210306078195572, + "val/num_eos_tokens": 20, + "val/ratio": 0.9886982440948486, + "val/ratio_var": 4.408901440910995e-05 + }, + { + "episode": 10576, + "epoch": 0.5341414141414141, + "eps": 0, + "loss/policy_avg": -0.01065884530544281, + "loss/value_avg": 0.23517704010009766, + "lr": 4.8664447703740905e-06, + "objective/entropy": -82.99937438964844, + "objective/kl": 39.04478073120117, + "objective/non_score_reward": -1.9522391557693481, + "objective/rlhf_reward": 14.461823463439941, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0008098644902929664, + "policy/clipfrac_avg": 0.00760706327855587, + "policy/entropy_avg": 0.15206003189086914, + "step": 661, + "val/clipfrac_avg": 0.13296082615852356, + "val/num_eos_tokens": 369, + "val/ratio": 0.9895342588424683, + "val/ratio_var": 4.8955425882013515e-05 + }, + { + "episode": 10592, + "epoch": 0.534949494949495, + "eps": 0, + "loss/policy_avg": 0.2483931928873062, + "loss/value_avg": 0.3836213946342468, + "lr": 4.853092917057224e-06, + "objective/entropy": -50.7081413269043, + "objective/kl": 33.77119827270508, + "objective/non_score_reward": -1.6885600090026855, + "objective/rlhf_reward": 13.561440467834473, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.00106827134732157, + "policy/clipfrac_avg": 0.005940725561231375, + "policy/entropy_avg": 0.18691349029541016, + "step": 662, + "val/clipfrac_avg": 0.08841180801391602, + "val/num_eos_tokens": 336, + "val/ratio": 0.9869168400764465, + "val/ratio_var": 0.00010277970432071015 + }, + { + "episode": 10608, + "epoch": 0.5357575757575758, + "eps": 0, + "loss/policy_avg": 0.018621079623699188, + "loss/value_avg": 0.47118768095970154, + "lr": 4.839742112141725e-06, + "objective/entropy": -22.153806686401367, + "objective/kl": 40.23773193359375, + "objective/non_score_reward": -2.0118865966796875, + "objective/rlhf_reward": 12.659988403320312, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.001191452844068408, + "policy/clipfrac_avg": 0.0037520253099501133, + "policy/entropy_avg": 0.2298755645751953, + "step": 663, + "val/clipfrac_avg": 0.13863703608512878, + "val/num_eos_tokens": 143, + "val/ratio": 0.984135627746582, + "val/ratio_var": 8.8629407400731e-05 + }, + { + "episode": 10624, + "epoch": 0.5365656565656566, + "eps": 0, + "loss/policy_avg": -0.1973285973072052, + "loss/value_avg": 0.8294652700424194, + "lr": 4.8263924509055185e-06, + "objective/entropy": -47.15129852294922, + "objective/kl": 32.980804443359375, + "objective/non_score_reward": -1.6490402221679688, + "objective/rlhf_reward": 12.577522277832031, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009336364455521107, + "policy/clipfrac_avg": 0.010629156604409218, + "policy/entropy_avg": 0.17932701110839844, + "step": 664, + "val/clipfrac_avg": 0.04402673989534378, + "val/num_eos_tokens": 230, + "val/ratio": 0.988549530506134, + "val/ratio_var": 7.59767135605216e-05 + }, + { + "episode": 10640, + "epoch": 0.5373737373737374, + "eps": 0, + "loss/policy_avg": -0.3642651438713074, + "loss/value_avg": 0.49642229080200195, + "lr": 4.8130440286183725e-06, + "objective/entropy": -92.09422302246094, + "objective/kl": 35.82035827636719, + "objective/non_score_reward": -1.7910178899765015, + "objective/rlhf_reward": 15.990232467651367, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0008464535931125283, + "policy/clipfrac_avg": 0.0052949972450733185, + "policy/entropy_avg": 0.16996145248413086, + "step": 665, + "val/clipfrac_avg": 0.18585266172885895, + "val/num_eos_tokens": 760, + "val/ratio": 0.9879733920097351, + "val/ratio_var": 7.15731002856046e-05 + }, + { + "episode": 10656, + "epoch": 0.5381818181818182, + "eps": 0, + "loss/policy_avg": 0.19750815629959106, + "loss/value_avg": 0.726028561592102, + "lr": 4.799696940541204e-06, + "objective/entropy": -100.74314880371094, + "objective/kl": 36.84406280517578, + "objective/non_score_reward": -1.842203140258789, + "objective/rlhf_reward": 13.720296859741211, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0008232034742832184, + "policy/clipfrac_avg": 0.005975116975605488, + "policy/entropy_avg": 0.17266035079956055, + "step": 666, + "val/clipfrac_avg": 0.0833563357591629, + "val/num_eos_tokens": 58, + "val/ratio": 0.9884688258171082, + "val/ratio_var": 5.402052556746639e-05 + }, + { + "episode": 10672, + "epoch": 0.538989898989899, + "eps": 0, + "loss/policy_avg": -0.3925812244415283, + "loss/value_avg": 0.9663629531860352, + "lr": 4.786351281925417e-06, + "objective/entropy": -124.33128356933594, + "objective/kl": 34.524009704589844, + "objective/non_score_reward": -1.7262005805969238, + "objective/rlhf_reward": 11.640987396240234, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007272096117958426, + "policy/clipfrac_avg": 0.008608815260231495, + "policy/entropy_avg": 0.14444184303283691, + "step": 667, + "val/clipfrac_avg": 0.03538993000984192, + "val/num_eos_tokens": 439, + "val/ratio": 0.9909723997116089, + "val/ratio_var": 4.3006653868360445e-05 + }, + { + "episode": 10688, + "epoch": 0.5397979797979798, + "eps": 0, + "loss/policy_avg": -0.06382110714912415, + "loss/value_avg": 0.3970024287700653, + "lr": 4.773007148012207e-06, + "objective/entropy": -34.56883239746094, + "objective/kl": 35.084346771240234, + "objective/non_score_reward": -1.7542171478271484, + "objective/rlhf_reward": 15.206720352172852, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011885869316756725, + "policy/clipfrac_avg": 0.006960581988096237, + "policy/entropy_avg": 0.198333740234375, + "step": 668, + "val/clipfrac_avg": 0.13442066311836243, + "val/num_eos_tokens": 187, + "val/ratio": 0.986121654510498, + "val/ratio_var": 3.918609945685603e-05 + }, + { + "episode": 10704, + "epoch": 0.5406060606060606, + "eps": 0, + "loss/policy_avg": 0.16306787729263306, + "loss/value_avg": 0.45442262291908264, + "lr": 4.759664634031897e-06, + "objective/entropy": -87.38048553466797, + "objective/kl": 34.169227600097656, + "objective/non_score_reward": -1.7084612846374512, + "objective/rlhf_reward": 12.05716323852539, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0009187318501062691, + "policy/clipfrac_avg": 0.005587933585047722, + "policy/entropy_avg": 0.17048358917236328, + "step": 669, + "val/clipfrac_avg": 0.19067704677581787, + "val/num_eos_tokens": 373, + "val/ratio": 0.9888883233070374, + "val/ratio_var": 6.463148747570813e-05 + }, + { + "episode": 10720, + "epoch": 0.5414141414141415, + "eps": 0, + "loss/policy_avg": 0.010326467454433441, + "loss/value_avg": 0.3474205434322357, + "lr": 4.74632383520324e-06, + "objective/entropy": -84.40043640136719, + "objective/kl": 36.53352355957031, + "objective/non_score_reward": -1.8266760110855103, + "objective/rlhf_reward": 13.681137084960938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009138257009908557, + "policy/clipfrac_avg": 0.006062377244234085, + "policy/entropy_avg": 0.1702251434326172, + "step": 670, + "val/clipfrac_avg": 0.09507843852043152, + "val/num_eos_tokens": 295, + "val/ratio": 0.9881380200386047, + "val/ratio_var": 4.7510071453871205e-05 + }, + { + "episode": 10736, + "epoch": 0.5422222222222223, + "eps": 0, + "loss/policy_avg": -0.09906017780303955, + "loss/value_avg": 0.8066251873970032, + "lr": 4.732984846732755e-06, + "objective/entropy": -81.22120666503906, + "objective/kl": 39.167991638183594, + "objective/non_score_reward": -1.9583998918533325, + "objective/rlhf_reward": 13.49472427368164, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009018934797495604, + "policy/clipfrac_avg": 0.015163691714406013, + "policy/entropy_avg": 0.17831039428710938, + "step": 671, + "val/clipfrac_avg": 0.0449674017727375, + "val/num_eos_tokens": 582, + "val/ratio": 0.988625168800354, + "val/ratio_var": 7.142542017390952e-05 + }, + { + "episode": 10752, + "epoch": 0.5430303030303031, + "eps": 0, + "loss/policy_avg": -0.2206135392189026, + "loss/value_avg": 0.23615601658821106, + "lr": 4.719647763814041e-06, + "objective/entropy": -144.14218139648438, + "objective/kl": 37.01219940185547, + "objective/non_score_reward": -1.8506097793579102, + "objective/rlhf_reward": 14.227514266967773, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0008922894485294819, + "policy/clipfrac_avg": 0.006483469158411026, + "policy/entropy_avg": 0.17594242095947266, + "step": 672, + "val/clipfrac_avg": 0.07687856256961823, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886537194252014, + "val/ratio_var": 6.714969640597701e-05 + }, + { + "episode": 10768, + "epoch": 0.5438383838383838, + "eps": 0, + "loss/policy_avg": -0.077633798122406, + "loss/value_avg": 0.6793819665908813, + "lr": 4.7063126816270965e-06, + "objective/entropy": -88.67999267578125, + "objective/kl": 39.09211349487305, + "objective/non_score_reward": -1.9546055793762207, + "objective/rlhf_reward": 13.686018943786621, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010067424736917019, + "policy/clipfrac_avg": 0.006908165290951729, + "policy/entropy_avg": 0.1997241973876953, + "step": 673, + "val/clipfrac_avg": 0.077333003282547, + "val/num_eos_tokens": 119, + "val/ratio": 0.9873216152191162, + "val/ratio_var": 6.026164919603616e-05 + }, + { + "episode": 10784, + "epoch": 0.5446464646464646, + "eps": 0, + "loss/policy_avg": 0.16710957884788513, + "loss/value_avg": 0.7977907657623291, + "lr": 4.6929796953376395e-06, + "objective/entropy": -60.31924819946289, + "objective/kl": 30.964773178100586, + "objective/non_score_reward": -1.548238754272461, + "objective/rlhf_reward": 12.936136245727539, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009357638191431761, + "policy/clipfrac_avg": 0.011794875375926495, + "policy/entropy_avg": 0.17830944061279297, + "step": 674, + "val/clipfrac_avg": 0.143951416015625, + "val/num_eos_tokens": 346, + "val/ratio": 0.9884299635887146, + "val/ratio_var": 9.555782162351534e-05 + }, + { + "episode": 10800, + "epoch": 0.5454545454545454, + "eps": 0, + "loss/policy_avg": -0.023821208626031876, + "loss/value_avg": 0.22992631793022156, + "lr": 4.679648900096436e-06, + "objective/entropy": -66.3176498413086, + "objective/kl": 32.99176788330078, + "objective/non_score_reward": -1.6495885848999023, + "objective/rlhf_reward": 14.381661415100098, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007904217927716672, + "policy/clipfrac_avg": 0.004523607436567545, + "policy/entropy_avg": 0.1616344451904297, + "step": 675, + "val/clipfrac_avg": 0.05289379507303238, + "val/num_eos_tokens": 387, + "val/ratio": 0.9895982146263123, + "val/ratio_var": 3.7571215216303244e-05 + }, + { + "episode": 10816, + "epoch": 0.5462626262626262, + "eps": 0, + "loss/policy_avg": 0.029874496161937714, + "loss/value_avg": 0.32752853631973267, + "lr": 4.666320391038612e-06, + "objective/entropy": -1.3176898956298828, + "objective/kl": 34.93087387084961, + "objective/non_score_reward": -1.7465436458587646, + "objective/rlhf_reward": 14.097206115722656, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0011160490103065968, + "policy/clipfrac_avg": 0.006563505623489618, + "policy/entropy_avg": 0.20054244995117188, + "step": 676, + "val/clipfrac_avg": 0.10146623104810715, + "val/num_eos_tokens": 224, + "val/ratio": 0.9860273003578186, + "val/ratio_var": 3.4691103792283684e-05 + }, + { + "episode": 10832, + "epoch": 0.547070707070707, + "eps": 0, + "loss/policy_avg": -0.10157568752765656, + "loss/value_avg": 0.4907248616218567, + "lr": 4.652994263282979e-06, + "objective/entropy": -96.90899658203125, + "objective/kl": 36.158023834228516, + "objective/non_score_reward": -1.8079012632369995, + "objective/rlhf_reward": 13.324911117553711, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0008718149038031697, + "policy/clipfrac_avg": 0.009348929859697819, + "policy/entropy_avg": 0.15552330017089844, + "step": 677, + "val/clipfrac_avg": 0.020003557205200195, + "val/num_eos_tokens": 119, + "val/ratio": 0.9896889925003052, + "val/ratio_var": 5.074829823570326e-05 + }, + { + "episode": 10848, + "epoch": 0.5478787878787879, + "eps": 0, + "loss/policy_avg": -0.263140469789505, + "loss/value_avg": 0.3839453458786011, + "lr": 4.6396706119313526e-06, + "objective/entropy": -82.16497802734375, + "objective/kl": 31.89101219177246, + "objective/non_score_reward": -1.5945507287979126, + "objective/rlhf_reward": 15.303886413574219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0011360663920640945, + "policy/clipfrac_avg": 0.007264793384820223, + "policy/entropy_avg": 0.1928253173828125, + "step": 678, + "val/clipfrac_avg": 0.09376704692840576, + "val/num_eos_tokens": 803, + "val/ratio": 0.9877759218215942, + "val/ratio_var": 5.5250733566936105e-05 + }, + { + "episode": 10864, + "epoch": 0.5486868686868687, + "eps": 0, + "loss/policy_avg": -0.2529081106185913, + "loss/value_avg": 1.246268391609192, + "lr": 4.626349532067879e-06, + "objective/entropy": -66.71647644042969, + "objective/kl": 37.956764221191406, + "objective/non_score_reward": -1.8978383541107178, + "objective/rlhf_reward": 14.820911407470703, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009447145857848227, + "policy/clipfrac_avg": 0.004722379148006439, + "policy/entropy_avg": 0.16110455989837646, + "step": 679, + "val/clipfrac_avg": 0.15111136436462402, + "val/num_eos_tokens": 376, + "val/ratio": 0.9897333979606628, + "val/ratio_var": 5.177267303224653e-05 + }, + { + "episode": 10880, + "epoch": 0.5494949494949495, + "eps": 0, + "loss/policy_avg": 0.2604403495788574, + "loss/value_avg": 0.9619290232658386, + "lr": 4.613031118758351e-06, + "objective/entropy": -96.83143615722656, + "objective/kl": 35.02674102783203, + "objective/non_score_reward": -1.7513370513916016, + "objective/rlhf_reward": 11.631475448608398, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0008016076171770692, + "policy/clipfrac_avg": 0.008088190108537674, + "policy/entropy_avg": 0.14316558837890625, + "step": 680, + "val/clipfrac_avg": 0.1873883306980133, + "val/num_eos_tokens": 55, + "val/ratio": 0.9903253316879272, + "val/ratio_var": 3.635772009147331e-05 + }, + { + "episode": 10896, + "epoch": 0.5503030303030303, + "eps": 0, + "loss/policy_avg": 0.20710928738117218, + "loss/value_avg": 0.8212553262710571, + "lr": 4.599715467049534e-06, + "objective/entropy": -46.69654846191406, + "objective/kl": 35.87485885620117, + "objective/non_score_reward": -1.7937428951263428, + "objective/rlhf_reward": 12.956256866455078, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0011159505229443312, + "policy/clipfrac_avg": 0.007098098285496235, + "policy/entropy_avg": 0.18874073028564453, + "step": 681, + "val/clipfrac_avg": 0.1026269719004631, + "val/num_eos_tokens": 32, + "val/ratio": 0.9870556592941284, + "val/ratio_var": 4.734138565254398e-05 + }, + { + "episode": 10912, + "epoch": 0.5511111111111111, + "eps": 0, + "loss/policy_avg": -0.19795550405979156, + "loss/value_avg": 0.36300960183143616, + "lr": 4.5864026719684796e-06, + "objective/entropy": -150.3234405517578, + "objective/kl": 45.23626708984375, + "objective/non_score_reward": -2.2618134021759033, + "objective/rlhf_reward": 12.62881088256836, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0013120709918439388, + "policy/clipfrac_avg": 0.0051262169145047665, + "policy/entropy_avg": 0.15036582946777344, + "step": 682, + "val/clipfrac_avg": 0.09302499890327454, + "val/num_eos_tokens": 451, + "val/ratio": 0.9899913668632507, + "val/ratio_var": 5.3359748562797904e-05 + }, + { + "episode": 10928, + "epoch": 0.5519191919191919, + "eps": 0, + "loss/policy_avg": 0.1822894662618637, + "loss/value_avg": 0.8603230118751526, + "lr": 4.573092828521857e-06, + "objective/entropy": -142.62588500976562, + "objective/kl": 31.69464111328125, + "objective/non_score_reward": -1.584732174873352, + "objective/rlhf_reward": 14.212142944335938, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.0007317900890484452, + "policy/clipfrac_avg": 0.01235207635909319, + "policy/entropy_avg": 0.13785886764526367, + "step": 683, + "val/clipfrac_avg": 0.14113879203796387, + "val/num_eos_tokens": 289, + "val/ratio": 0.9909247159957886, + "val/ratio_var": 5.3253872465575114e-05 + }, + { + "episode": 10944, + "epoch": 0.5527272727272727, + "eps": 0, + "loss/policy_avg": -0.0519242025911808, + "loss/value_avg": 0.19368618726730347, + "lr": 4.559786031695275e-06, + "objective/entropy": -27.279052734375, + "objective/kl": 43.83638000488281, + "objective/non_score_reward": -2.191819190979004, + "objective/rlhf_reward": 14.433180809020996, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010979644721373916, + "policy/clipfrac_avg": 0.005808757618069649, + "policy/entropy_avg": 0.21721267700195312, + "step": 684, + "val/clipfrac_avg": 0.00993678905069828, + "val/num_eos_tokens": 24, + "val/ratio": 0.9861806631088257, + "val/ratio_var": 3.975445360993035e-05 + }, + { + "episode": 10960, + "epoch": 0.5535353535353535, + "eps": 0, + "loss/policy_avg": 0.11247897148132324, + "loss/value_avg": 0.3687130808830261, + "lr": 4.546482376452589e-06, + "objective/entropy": -114.476806640625, + "objective/kl": 26.137264251708984, + "objective/non_score_reward": -1.3068633079528809, + "objective/rlhf_reward": 12.943137168884277, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0007893022848293185, + "policy/clipfrac_avg": 0.0076246787793934345, + "policy/entropy_avg": 0.14629364013671875, + "step": 685, + "val/clipfrac_avg": 0.0864788144826889, + "val/num_eos_tokens": 446, + "val/ratio": 0.9891906976699829, + "val/ratio_var": 7.146107236621901e-05 + }, + { + "episode": 10976, + "epoch": 0.5543434343434344, + "eps": 0, + "loss/policy_avg": -0.035443827509880066, + "loss/value_avg": 0.3387452960014343, + "lr": 4.533181957735247e-06, + "objective/entropy": -129.17437744140625, + "objective/kl": 28.205734252929688, + "objective/non_score_reward": -1.4102866649627686, + "objective/rlhf_reward": 11.988150596618652, + "objective/scores": 13.375, + "policy/approxkl_avg": 0.0007981459493748844, + "policy/clipfrac_avg": 0.004403797909617424, + "policy/entropy_avg": 0.15001487731933594, + "step": 686, + "val/clipfrac_avg": 0.05607829615473747, + "val/num_eos_tokens": 331, + "val/ratio": 0.9896979331970215, + "val/ratio_var": 4.439734038896859e-05 + }, + { + "episode": 10992, + "epoch": 0.5551515151515152, + "eps": 0, + "loss/policy_avg": 0.03433602303266525, + "loss/value_avg": 0.7930219173431396, + "lr": 4.5198848704615915e-06, + "objective/entropy": -69.12667083740234, + "objective/kl": 34.03411865234375, + "objective/non_score_reward": -1.7017061710357666, + "objective/rlhf_reward": 13.399856567382812, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0009728028671815991, + "policy/clipfrac_avg": 0.005738085135817528, + "policy/entropy_avg": 0.19457244873046875, + "step": 687, + "val/clipfrac_avg": 0.11831800639629364, + "val/num_eos_tokens": 44, + "val/ratio": 0.9865483045578003, + "val/ratio_var": 7.133706822060049e-05 + }, + { + "episode": 11008, + "epoch": 0.555959595959596, + "eps": 0, + "loss/policy_avg": -0.22896869480609894, + "loss/value_avg": 0.9052561521530151, + "lr": 4.506591209526195e-06, + "objective/entropy": -71.97180938720703, + "objective/kl": 30.15737533569336, + "objective/non_score_reward": -1.507868766784668, + "objective/rlhf_reward": 13.203069686889648, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0008744328515604138, + "policy/clipfrac_avg": 0.005093730054795742, + "policy/entropy_avg": 0.15955305099487305, + "step": 688, + "val/clipfrac_avg": 0.08035694062709808, + "val/num_eos_tokens": 290, + "val/ratio": 0.9889332056045532, + "val/ratio_var": 3.423935777391307e-05 + }, + { + "episode": 11024, + "epoch": 0.5567676767676768, + "eps": 0, + "loss/policy_avg": 0.03792828321456909, + "loss/value_avg": 0.9123845100402832, + "lr": 4.493301069799171e-06, + "objective/entropy": -78.7776107788086, + "objective/kl": 33.093013763427734, + "objective/non_score_reward": -1.6546508073806763, + "objective/rlhf_reward": 14.282849311828613, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0009888019412755966, + "policy/clipfrac_avg": 0.00978151336312294, + "policy/entropy_avg": 0.17550277709960938, + "step": 689, + "val/clipfrac_avg": 0.07123228162527084, + "val/num_eos_tokens": 16, + "val/ratio": 0.9879224300384521, + "val/ratio_var": 6.044195106369443e-05 + }, + { + "episode": 11040, + "epoch": 0.5575757575757576, + "eps": 0, + "loss/policy_avg": 0.015539663843810558, + "loss/value_avg": 0.3836192786693573, + "lr": 4.480014546125511e-06, + "objective/entropy": 5.367833614349365, + "objective/kl": 33.34143829345703, + "objective/non_score_reward": -1.667072057723999, + "objective/rlhf_reward": 13.621990203857422, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.001577872084453702, + "policy/clipfrac_avg": 0.009263614192605019, + "policy/entropy_avg": 0.2828826904296875, + "step": 690, + "val/clipfrac_avg": 0.05502686649560928, + "val/num_eos_tokens": 182, + "val/ratio": 0.9818495512008667, + "val/ratio_var": 7.278368866536766e-05 + }, + { + "episode": 11056, + "epoch": 0.5583838383838384, + "eps": 0, + "loss/policy_avg": -0.0511251725256443, + "loss/value_avg": 0.38562020659446716, + "lr": 4.466731733324399e-06, + "objective/entropy": -61.70869064331055, + "objective/kl": 33.737091064453125, + "objective/non_score_reward": -1.686854600906372, + "objective/rlhf_reward": 14.25845718383789, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0008769157575443387, + "policy/clipfrac_avg": 0.0052274661138653755, + "policy/entropy_avg": 0.17685508728027344, + "step": 691, + "val/clipfrac_avg": 0.032033294439315796, + "val/num_eos_tokens": 406, + "val/ratio": 0.9887614250183105, + "val/ratio_var": 4.9923761253012344e-05 + }, + { + "episode": 11072, + "epoch": 0.5591919191919192, + "eps": 0, + "loss/policy_avg": 0.29204708337783813, + "loss/value_avg": 0.5752546787261963, + "lr": 4.4534527261885365e-06, + "objective/entropy": -69.50019836425781, + "objective/kl": 31.425121307373047, + "objective/non_score_reward": -1.571256160736084, + "objective/rlhf_reward": 12.920930862426758, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.000965589948464185, + "policy/clipfrac_avg": 0.009000038728117943, + "policy/entropy_avg": 0.18520164489746094, + "step": 692, + "val/clipfrac_avg": 0.09444969892501831, + "val/num_eos_tokens": 42, + "val/ratio": 0.9867506623268127, + "val/ratio_var": 9.781523112906143e-05 + }, + { + "episode": 11088, + "epoch": 0.56, + "eps": 0, + "loss/policy_avg": -0.304983526468277, + "loss/value_avg": 0.4396636188030243, + "lr": 4.4401776194834615e-06, + "objective/entropy": -95.70045471191406, + "objective/kl": 31.978641510009766, + "objective/non_score_reward": -1.5989320278167725, + "objective/rlhf_reward": 13.791692733764648, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0008498022216372192, + "policy/clipfrac_avg": 0.014874793589115143, + "policy/entropy_avg": 0.15596485137939453, + "step": 693, + "val/clipfrac_avg": 0.08384636044502258, + "val/num_eos_tokens": 116, + "val/ratio": 0.989993691444397, + "val/ratio_var": 7.268232002388686e-05 + }, + { + "episode": 11104, + "epoch": 0.5608080808080808, + "eps": 0, + "loss/policy_avg": -0.28100529313087463, + "loss/value_avg": 0.36995184421539307, + "lr": 4.426906507946881e-06, + "objective/entropy": -68.41473388671875, + "objective/kl": 29.767484664916992, + "objective/non_score_reward": -1.4883742332458496, + "objective/rlhf_reward": 12.292876243591309, + "objective/scores": 13.75, + "policy/approxkl_avg": 0.0010167569853365421, + "policy/clipfrac_avg": 0.01529286801815033, + "policy/entropy_avg": 0.18943500518798828, + "step": 694, + "val/clipfrac_avg": 0.14090892672538757, + "val/num_eos_tokens": 18, + "val/ratio": 0.9882405400276184, + "val/ratio_var": 9.475964907323942e-05 + }, + { + "episode": 11120, + "epoch": 0.5616161616161616, + "eps": 0, + "loss/policy_avg": -0.06622211635112762, + "loss/value_avg": 0.334886372089386, + "lr": 4.413639486287992e-06, + "objective/entropy": -12.017631530761719, + "objective/kl": 32.63812255859375, + "objective/non_score_reward": -1.6319061517715454, + "objective/rlhf_reward": 15.243093490600586, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0010099872015416622, + "policy/clipfrac_avg": 0.004084677901118994, + "policy/entropy_avg": 0.18613624572753906, + "step": 695, + "val/clipfrac_avg": 0.062321148812770844, + "val/num_eos_tokens": 267, + "val/ratio": 0.987127423286438, + "val/ratio_var": 4.299115244066343e-05 + }, + { + "episode": 11136, + "epoch": 0.5624242424242424, + "eps": 0, + "loss/policy_avg": -0.06522263586521149, + "loss/value_avg": 0.7103580236434937, + "lr": 4.4003766491867984e-06, + "objective/entropy": -87.83956909179688, + "objective/kl": 28.685632705688477, + "objective/non_score_reward": -1.4342817068099976, + "objective/rlhf_reward": 11.417280197143555, + "objective/scores": 12.875, + "policy/approxkl_avg": 0.0008759163902141154, + "policy/clipfrac_avg": 0.006767658516764641, + "policy/entropy_avg": 0.19121313095092773, + "step": 696, + "val/clipfrac_avg": 0.18274252116680145, + "val/num_eos_tokens": 573, + "val/ratio": 0.9897185564041138, + "val/ratio_var": 5.892527042306028e-05 + }, + { + "episode": 11152, + "epoch": 0.5632323232323232, + "eps": 0, + "loss/policy_avg": 0.1779220849275589, + "loss/value_avg": 0.6948210000991821, + "lr": 4.3871180912934465e-06, + "objective/entropy": -99.48849487304688, + "objective/kl": 33.15848159790039, + "objective/non_score_reward": -1.6579241752624512, + "objective/rlhf_reward": 11.59207534790039, + "objective/scores": 13.25, + "policy/approxkl_avg": 0.0008881977992132306, + "policy/clipfrac_avg": 0.007622114382684231, + "policy/entropy_avg": 0.16399192810058594, + "step": 697, + "val/clipfrac_avg": 0.13009925186634064, + "val/num_eos_tokens": 59, + "val/ratio": 0.9894096255302429, + "val/ratio_var": 4.7340425226138905e-05 + }, + { + "episode": 11168, + "epoch": 0.564040404040404, + "eps": 0, + "loss/policy_avg": 0.22691625356674194, + "loss/value_avg": 0.518365740776062, + "lr": 4.373863907227539e-06, + "objective/entropy": -121.6069564819336, + "objective/kl": 28.16010093688965, + "objective/non_score_reward": -1.4080051183700562, + "objective/rlhf_reward": 14.021682739257812, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0007813716074451804, + "policy/clipfrac_avg": 0.010615997016429901, + "policy/entropy_avg": 0.1465442180633545, + "step": 698, + "val/clipfrac_avg": 0.04978542774915695, + "val/num_eos_tokens": 567, + "val/ratio": 0.9906186461448669, + "val/ratio_var": 7.280353020178154e-05 + }, + { + "episode": 11184, + "epoch": 0.5648484848484848, + "eps": 0, + "loss/policy_avg": 0.4753793179988861, + "loss/value_avg": 0.8896934390068054, + "lr": 4.3606141915774695e-06, + "objective/entropy": -56.37757110595703, + "objective/kl": 29.718286514282227, + "objective/non_score_reward": -1.4859144687652588, + "objective/rlhf_reward": 14.08439826965332, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.0009621034259907901, + "policy/clipfrac_avg": 0.011594917625188828, + "policy/entropy_avg": 0.15114402770996094, + "step": 699, + "val/clipfrac_avg": 0.028925057500600815, + "val/num_eos_tokens": 93, + "val/ratio": 0.9886069297790527, + "val/ratio_var": 5.0333743274677545e-05 + }, + { + "episode": 11200, + "epoch": 0.5656565656565656, + "eps": 0, + "loss/policy_avg": 0.013627409934997559, + "loss/value_avg": 0.39067256450653076, + "lr": 4.347369038899744e-06, + "objective/entropy": -51.717899322509766, + "objective/kl": 30.32842445373535, + "objective/non_score_reward": -1.5164213180541992, + "objective/rlhf_reward": 11.1085786819458, + "objective/scores": 12.625, + "policy/approxkl_avg": 0.0010908711701631546, + "policy/clipfrac_avg": 0.005067873280495405, + "policy/entropy_avg": 0.21241378784179688, + "step": 700, + "val/clipfrac_avg": 0.05506103113293648, + "val/num_eos_tokens": 84, + "val/ratio": 0.9856171607971191, + "val/ratio_var": 8.688835805514827e-05 + }, + { + "episode": 11216, + "epoch": 0.5664646464646464, + "eps": 0, + "loss/policy_avg": -0.09055965393781662, + "loss/value_avg": 0.5455071926116943, + "lr": 4.334128543718295e-06, + "objective/entropy": -74.95745849609375, + "objective/kl": 33.42132568359375, + "objective/non_score_reward": -1.6710662841796875, + "objective/rlhf_reward": 14.828933715820312, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0013184514828026295, + "policy/clipfrac_avg": 0.00809475127607584, + "policy/entropy_avg": 0.17902374267578125, + "step": 701, + "val/clipfrac_avg": 0.18833743035793304, + "val/num_eos_tokens": 358, + "val/ratio": 0.9879262447357178, + "val/ratio_var": 6.120221951277927e-05 + }, + { + "episode": 11232, + "epoch": 0.5672727272727273, + "eps": 0, + "loss/policy_avg": 0.2261451780796051, + "loss/value_avg": 1.5403046607971191, + "lr": 4.320892800523827e-06, + "objective/entropy": -90.61648559570312, + "objective/kl": 27.537485122680664, + "objective/non_score_reward": -1.3768744468688965, + "objective/rlhf_reward": 12.826250076293945, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009536159923300147, + "policy/clipfrac_avg": 0.01545755285769701, + "policy/entropy_avg": 0.20562219619750977, + "step": 702, + "val/clipfrac_avg": 0.033450622111558914, + "val/num_eos_tokens": 595, + "val/ratio": 0.9873440265655518, + "val/ratio_var": 7.613677007611841e-05 + }, + { + "episode": 11248, + "epoch": 0.5680808080808081, + "eps": 0, + "loss/policy_avg": -0.05153674632310867, + "loss/value_avg": 0.18954747915267944, + "lr": 4.307661903773129e-06, + "objective/entropy": -56.119659423828125, + "objective/kl": 43.216548919677734, + "objective/non_score_reward": -2.160827398300171, + "objective/rlhf_reward": 15.80792236328125, + "objective/scores": 18.0, + "policy/approxkl_avg": 0.0010682737920433283, + "policy/clipfrac_avg": 0.005114596337080002, + "policy/entropy_avg": 0.20781707763671875, + "step": 703, + "val/clipfrac_avg": 0.12178023159503937, + "val/num_eos_tokens": 244, + "val/ratio": 0.9860707521438599, + "val/ratio_var": 5.392924140323885e-05 + }, + { + "episode": 11264, + "epoch": 0.5688888888888889, + "eps": 0, + "loss/policy_avg": 0.22105136513710022, + "loss/value_avg": 0.43452760577201843, + "lr": 4.294435947888405e-06, + "objective/entropy": -180.00924682617188, + "objective/kl": 32.27890396118164, + "objective/non_score_reward": -1.6139451265335083, + "objective/rlhf_reward": 12.425117492675781, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007389815291389823, + "policy/clipfrac_avg": 0.00613048393279314, + "policy/entropy_avg": 0.15887287259101868, + "step": 704, + "val/clipfrac_avg": 0.12273523956537247, + "val/num_eos_tokens": 611, + "val/ratio": 0.9900252819061279, + "val/ratio_var": 6.185931124491617e-05 + }, + { + "episode": 11280, + "epoch": 0.5696969696969697, + "eps": 0, + "loss/policy_avg": -0.044020622968673706, + "loss/value_avg": 1.459969401359558, + "lr": 4.281215027256592e-06, + "objective/entropy": -73.2503890991211, + "objective/kl": 29.207317352294922, + "objective/non_score_reward": -1.460365891456604, + "objective/rlhf_reward": 15.766195297241211, + "objective/scores": 17.25, + "policy/approxkl_avg": 0.0010228066239506006, + "policy/clipfrac_avg": 0.010731351561844349, + "policy/entropy_avg": 0.17015647888183594, + "step": 705, + "val/clipfrac_avg": 0.1418685019016266, + "val/num_eos_tokens": 378, + "val/ratio": 0.9865360856056213, + "val/ratio_var": 0.0001155980207840912 + }, + { + "episode": 11296, + "epoch": 0.5705050505050505, + "eps": 0, + "loss/policy_avg": 0.10734500735998154, + "loss/value_avg": 0.40052735805511475, + "lr": 4.2679992362287e-06, + "objective/entropy": -91.05632019042969, + "objective/kl": 39.329261779785156, + "objective/non_score_reward": -1.966463327407837, + "objective/rlhf_reward": 12.744474411010742, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0010600072564557195, + "policy/clipfrac_avg": 0.005786027293652296, + "policy/entropy_avg": 0.23768234252929688, + "step": 706, + "val/clipfrac_avg": 0.032500892877578735, + "val/num_eos_tokens": 203, + "val/ratio": 0.9860942959785461, + "val/ratio_var": 5.682241317117587e-05 + }, + { + "episode": 11312, + "epoch": 0.5713131313131313, + "eps": 0, + "loss/policy_avg": -0.5062837600708008, + "loss/value_avg": 0.7742470502853394, + "lr": 4.254788669119127e-06, + "objective/entropy": -93.2535400390625, + "objective/kl": 34.540130615234375, + "objective/non_score_reward": -1.727006435394287, + "objective/rlhf_reward": 14.788618087768555, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008475214708596468, + "policy/clipfrac_avg": 0.027025574818253517, + "policy/entropy_avg": 0.21597671508789062, + "step": 707, + "val/clipfrac_avg": 0.1775030940771103, + "val/num_eos_tokens": 165, + "val/ratio": 0.9894334077835083, + "val/ratio_var": 9.222733206115663e-05 + }, + { + "episode": 11328, + "epoch": 0.5721212121212121, + "eps": 0, + "loss/policy_avg": -0.1463669091463089, + "loss/value_avg": 0.5632686614990234, + "lr": 4.241583420204998e-06, + "objective/entropy": -110.37503051757812, + "objective/kl": 36.62909698486328, + "objective/non_score_reward": -1.831454873085022, + "objective/rlhf_reward": 12.89510726928711, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0009251610608771443, + "policy/clipfrac_avg": 0.00941617414355278, + "policy/entropy_avg": 0.16801166534423828, + "step": 708, + "val/clipfrac_avg": 0.04514690116047859, + "val/num_eos_tokens": 267, + "val/ratio": 0.9897239208221436, + "val/ratio_var": 7.692311191931367e-05 + }, + { + "episode": 11344, + "epoch": 0.572929292929293, + "eps": 0, + "loss/policy_avg": -0.35295569896698, + "loss/value_avg": 0.7035583257675171, + "lr": 4.228383583725471e-06, + "objective/entropy": -76.27204132080078, + "objective/kl": 31.900005340576172, + "objective/non_score_reward": -1.5950002670288086, + "objective/rlhf_reward": 14.053436279296875, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0010020419722422957, + "policy/clipfrac_avg": 0.011507621966302395, + "policy/entropy_avg": 0.16324424743652344, + "step": 709, + "val/clipfrac_avg": 0.12508229911327362, + "val/num_eos_tokens": 173, + "val/ratio": 0.9896982908248901, + "val/ratio_var": 4.9723090342013165e-05 + }, + { + "episode": 11360, + "epoch": 0.5737373737373738, + "eps": 0, + "loss/policy_avg": 0.160404771566391, + "loss/value_avg": 1.1914963722229004, + "lr": 4.215189253881093e-06, + "objective/entropy": -79.55085754394531, + "objective/kl": 36.81852722167969, + "objective/non_score_reward": -1.8409266471862793, + "objective/rlhf_reward": 12.159073829650879, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0011080782860517502, + "policy/clipfrac_avg": 0.006503967102617025, + "policy/entropy_avg": 0.19612884521484375, + "step": 710, + "val/clipfrac_avg": 0.08140315115451813, + "val/num_eos_tokens": 538, + "val/ratio": 0.9866147041320801, + "val/ratio_var": 7.869482942624018e-05 + }, + { + "episode": 11376, + "epoch": 0.5745454545454546, + "eps": 0, + "loss/policy_avg": 0.07548795640468597, + "loss/value_avg": 0.5138013362884521, + "lr": 4.2020005248331056e-06, + "objective/entropy": -119.03164672851562, + "objective/kl": 34.31198501586914, + "objective/non_score_reward": -1.7155992984771729, + "objective/rlhf_reward": 13.503150939941406, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0007362160831689835, + "policy/clipfrac_avg": 0.00589254405349493, + "policy/entropy_avg": 0.14674091339111328, + "step": 711, + "val/clipfrac_avg": 0.02668866142630577, + "val/num_eos_tokens": 515, + "val/ratio": 0.990146279335022, + "val/ratio_var": 3.9673428545938805e-05 + }, + { + "episode": 11392, + "epoch": 0.5753535353535354, + "eps": 0, + "loss/policy_avg": -0.2925986051559448, + "loss/value_avg": 0.3821415603160858, + "lr": 4.1888174907027775e-06, + "objective/entropy": -112.06816101074219, + "objective/kl": 38.006500244140625, + "objective/non_score_reward": -1.900325059890747, + "objective/rlhf_reward": 13.990300178527832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.000874843040946871, + "policy/clipfrac_avg": 0.007544378750026226, + "policy/entropy_avg": 0.16875195503234863, + "step": 712, + "val/clipfrac_avg": 0.025153353810310364, + "val/num_eos_tokens": 4, + "val/ratio": 0.9893897771835327, + "val/ratio_var": 8.519255788996816e-05 + }, + { + "episode": 11408, + "epoch": 0.5761616161616162, + "eps": 0, + "loss/policy_avg": -0.5444605350494385, + "loss/value_avg": 1.517474889755249, + "lr": 4.175640245570742e-06, + "objective/entropy": -92.28114318847656, + "objective/kl": 35.69511413574219, + "objective/non_score_reward": -1.7847557067871094, + "objective/rlhf_reward": 13.73868179321289, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008441185345873237, + "policy/clipfrac_avg": 0.005670716054737568, + "policy/entropy_avg": 0.1591033935546875, + "step": 713, + "val/clipfrac_avg": 0.1791025698184967, + "val/num_eos_tokens": 98, + "val/ratio": 0.9903852939605713, + "val/ratio_var": 4.043114677187987e-05 + }, + { + "episode": 11424, + "epoch": 0.576969696969697, + "eps": 0, + "loss/policy_avg": -0.3059786260128021, + "loss/value_avg": 0.6795393228530884, + "lr": 4.162468883476319e-06, + "objective/entropy": -193.77468872070312, + "objective/kl": 42.17440414428711, + "objective/non_score_reward": -2.108720302581787, + "objective/rlhf_reward": 12.281905174255371, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0008298080647364259, + "policy/clipfrac_avg": 0.00635810662060976, + "policy/entropy_avg": 0.15823698043823242, + "step": 714, + "val/clipfrac_avg": 0.0064142923802137375, + "val/num_eos_tokens": 1666, + "val/ratio": 0.9892975091934204, + "val/ratio_var": 6.228621350601315e-05 + }, + { + "episode": 11440, + "epoch": 0.5777777777777777, + "eps": 0, + "loss/policy_avg": -0.023257479071617126, + "loss/value_avg": 0.18199321627616882, + "lr": 4.149303498416838e-06, + "objective/entropy": -22.63348388671875, + "objective/kl": 44.308345794677734, + "objective/non_score_reward": -2.2154173851013184, + "objective/rlhf_reward": 14.82364559173584, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0012546201469376683, + "policy/clipfrac_avg": 0.007735411170870066, + "policy/entropy_avg": 0.240936279296875, + "step": 715, + "val/clipfrac_avg": 0.0028473599813878536, + "val/num_eos_tokens": 302, + "val/ratio": 0.9840776920318604, + "val/ratio_var": 5.949061960563995e-05 + }, + { + "episode": 11456, + "epoch": 0.5785858585858585, + "eps": 0, + "loss/policy_avg": 0.3237575888633728, + "loss/value_avg": 0.38197559118270874, + "lr": 4.136144184346979e-06, + "objective/entropy": -168.4746856689453, + "objective/kl": 40.63113784790039, + "objective/non_score_reward": -2.031557083129883, + "objective/rlhf_reward": 13.070005416870117, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.0007934231543913484, + "policy/clipfrac_avg": 0.012050900608301163, + "policy/entropy_avg": 0.14413070678710938, + "step": 716, + "val/clipfrac_avg": 0.09322839975357056, + "val/num_eos_tokens": 1174, + "val/ratio": 0.9900047779083252, + "val/ratio_var": 5.29003118572291e-05 + }, + { + "episode": 11472, + "epoch": 0.5793939393939394, + "eps": 0, + "loss/policy_avg": 0.23654325306415558, + "loss/value_avg": 1.166879653930664, + "lr": 4.122991035178093e-06, + "objective/entropy": -96.77055358886719, + "objective/kl": 42.17526626586914, + "objective/non_score_reward": -2.1087632179260254, + "objective/rlhf_reward": 14.500611305236816, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.000815618084743619, + "policy/clipfrac_avg": 0.008080050349235535, + "policy/entropy_avg": 0.14806270599365234, + "step": 717, + "val/clipfrac_avg": 0.02973807230591774, + "val/num_eos_tokens": 297, + "val/ratio": 0.9899397492408752, + "val/ratio_var": 3.871042645187117e-05 + }, + { + "episode": 11488, + "epoch": 0.5802020202020202, + "eps": 0, + "loss/policy_avg": -0.11968575417995453, + "loss/value_avg": 0.9426589608192444, + "lr": 4.109844144777537e-06, + "objective/entropy": -58.205047607421875, + "objective/kl": 42.6419792175293, + "objective/non_score_reward": -2.132099151611328, + "objective/rlhf_reward": 12.203838348388672, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0009566791704855859, + "policy/clipfrac_avg": 0.0025268401950597763, + "policy/entropy_avg": 0.18893003463745117, + "step": 718, + "val/clipfrac_avg": 0.1098824143409729, + "val/num_eos_tokens": 16, + "val/ratio": 0.9869214296340942, + "val/ratio_var": 8.384165266761556e-05 + }, + { + "episode": 11504, + "epoch": 0.581010101010101, + "eps": 0, + "loss/policy_avg": 0.0208362378180027, + "loss/value_avg": 0.8499202728271484, + "lr": 4.096703606968007e-06, + "objective/entropy": -105.71292114257812, + "objective/kl": 40.70527648925781, + "objective/non_score_reward": -2.035263776779175, + "objective/rlhf_reward": 13.605360984802246, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008132618968375027, + "policy/clipfrac_avg": 0.004117591306567192, + "policy/entropy_avg": 0.15980243682861328, + "step": 719, + "val/clipfrac_avg": 0.04368842393159866, + "val/num_eos_tokens": 428, + "val/ratio": 0.9895676374435425, + "val/ratio_var": 6.342094275169075e-05 + }, + { + "episode": 11520, + "epoch": 0.5818181818181818, + "eps": 0, + "loss/policy_avg": -0.11627452820539474, + "loss/value_avg": 0.7368168830871582, + "lr": 4.083569515526853e-06, + "objective/entropy": -100.23733520507812, + "objective/kl": 32.93721008300781, + "objective/non_score_reward": -1.6468604803085327, + "objective/rlhf_reward": 15.134389877319336, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0008385346736758947, + "policy/clipfrac_avg": 0.009182492271065712, + "policy/entropy_avg": 0.17890548706054688, + "step": 720, + "val/clipfrac_avg": 0.06867726147174835, + "val/num_eos_tokens": 144, + "val/ratio": 0.9896779656410217, + "val/ratio_var": 7.192800694610924e-05 + }, + { + "episode": 11536, + "epoch": 0.5826262626262626, + "eps": 0, + "loss/policy_avg": -0.06581735610961914, + "loss/value_avg": 0.7933100461959839, + "lr": 4.070441964185428e-06, + "objective/entropy": -103.3985366821289, + "objective/kl": 39.985321044921875, + "objective/non_score_reward": -1.9992659091949463, + "objective/rlhf_reward": 14.547609329223633, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000881620217114687, + "policy/clipfrac_avg": 0.004334163852035999, + "policy/entropy_avg": 0.1855926513671875, + "step": 721, + "val/clipfrac_avg": 0.04114583134651184, + "val/num_eos_tokens": 161, + "val/ratio": 0.989449679851532, + "val/ratio_var": 6.215622852323577e-05 + }, + { + "episode": 11552, + "epoch": 0.5834343434343434, + "eps": 0, + "loss/policy_avg": 0.44041353464126587, + "loss/value_avg": 0.7198789119720459, + "lr": 4.05732104662841e-06, + "objective/entropy": -119.77040100097656, + "objective/kl": 36.42585754394531, + "objective/non_score_reward": -1.8212929964065552, + "objective/rlhf_reward": 13.467769622802734, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008070225012488663, + "policy/clipfrac_avg": 0.007022568956017494, + "policy/entropy_avg": 0.15318524837493896, + "step": 722, + "val/clipfrac_avg": 0.05163312703371048, + "val/num_eos_tokens": 304, + "val/ratio": 0.9899815320968628, + "val/ratio_var": 4.889031333732419e-05 + }, + { + "episode": 11568, + "epoch": 0.5842424242424242, + "eps": 0, + "loss/policy_avg": 0.4866091012954712, + "loss/value_avg": 1.1953067779541016, + "lr": 4.04420685649314e-06, + "objective/entropy": -50.46599578857422, + "objective/kl": 39.113037109375, + "objective/non_score_reward": -1.955651879310608, + "objective/rlhf_reward": 13.911535263061523, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0010366745991632342, + "policy/clipfrac_avg": 0.006084335036575794, + "policy/entropy_avg": 0.20149445533752441, + "step": 723, + "val/clipfrac_avg": 0.09489050507545471, + "val/num_eos_tokens": 36, + "val/ratio": 0.9867548942565918, + "val/ratio_var": 6.938407750567421e-05 + }, + { + "episode": 11584, + "epoch": 0.585050505050505, + "eps": 0, + "loss/policy_avg": 0.1589544266462326, + "loss/value_avg": 0.568170964717865, + "lr": 4.031099487368937e-06, + "objective/entropy": -112.03081512451172, + "objective/kl": 37.355255126953125, + "objective/non_score_reward": -1.8677629232406616, + "objective/rlhf_reward": 13.49161148071289, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.001035548048093915, + "policy/clipfrac_avg": 0.006865138188004494, + "policy/entropy_avg": 0.20564031600952148, + "step": 724, + "val/clipfrac_avg": 0.05233292281627655, + "val/num_eos_tokens": 347, + "val/ratio": 0.9859846830368042, + "val/ratio_var": 0.00010150864545721561 + }, + { + "episode": 11600, + "epoch": 0.5858585858585859, + "eps": 0, + "loss/policy_avg": -0.19635428488254547, + "loss/value_avg": 0.33023789525032043, + "lr": 4.0179990327964526e-06, + "objective/entropy": -69.79931640625, + "objective/kl": 41.90481948852539, + "objective/non_score_reward": -2.095241069793701, + "objective/rlhf_reward": 14.85007095336914, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0010360007872805, + "policy/clipfrac_avg": 0.004560987465083599, + "policy/entropy_avg": 0.20290184020996094, + "step": 725, + "val/clipfrac_avg": 0.08375490456819534, + "val/num_eos_tokens": 261, + "val/ratio": 0.98649001121521, + "val/ratio_var": 6.110723916208372e-05 + }, + { + "episode": 11616, + "epoch": 0.5866666666666667, + "eps": 0, + "loss/policy_avg": 0.0999169647693634, + "loss/value_avg": 1.0625457763671875, + "lr": 4.004905586266988e-06, + "objective/entropy": -104.75285339355469, + "objective/kl": 32.01334762573242, + "objective/non_score_reward": -1.6006674766540527, + "objective/rlhf_reward": 12.766520500183105, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010034965816885233, + "policy/clipfrac_avg": 0.007322699762880802, + "policy/entropy_avg": 0.19988441467285156, + "step": 726, + "val/clipfrac_avg": 0.14562752842903137, + "val/num_eos_tokens": 33, + "val/ratio": 0.9887305498123169, + "val/ratio_var": 7.558859942946583e-05 + }, + { + "episode": 11632, + "epoch": 0.5874747474747475, + "eps": 0, + "loss/policy_avg": 0.004431502893567085, + "loss/value_avg": 0.5651569366455078, + "lr": 3.991819241221836e-06, + "objective/entropy": -53.65510940551758, + "objective/kl": 43.23419952392578, + "objective/non_score_reward": -2.161710262298584, + "objective/rlhf_reward": 12.986727714538574, + "objective/scores": 15.125, + "policy/approxkl_avg": 0.001010584644973278, + "policy/clipfrac_avg": 0.006680442951619625, + "policy/entropy_avg": 0.17978286743164062, + "step": 727, + "val/clipfrac_avg": 0.028638603165745735, + "val/num_eos_tokens": 25, + "val/ratio": 0.9880765676498413, + "val/ratio_var": 3.508182999212295e-05 + }, + { + "episode": 11648, + "epoch": 0.5882828282828283, + "eps": 0, + "loss/policy_avg": -0.026804931461811066, + "loss/value_avg": 0.3243325352668762, + "lr": 3.978740091051599e-06, + "objective/entropy": -147.60580444335938, + "objective/kl": 40.000404357910156, + "objective/non_score_reward": -2.0000202655792236, + "objective/rlhf_reward": 13.304666519165039, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0008220034651458263, + "policy/clipfrac_avg": 0.010559421963989735, + "policy/entropy_avg": 0.15192937850952148, + "step": 728, + "val/clipfrac_avg": 0.047058261930942535, + "val/num_eos_tokens": 93, + "val/ratio": 0.9890267252922058, + "val/ratio_var": 7.159873348427936e-05 + }, + { + "episode": 11664, + "epoch": 0.5890909090909091, + "eps": 0, + "loss/policy_avg": 0.061702877283096313, + "loss/value_avg": 0.25130608677864075, + "lr": 3.965668229095546e-06, + "objective/entropy": -75.26506042480469, + "objective/kl": 34.32489013671875, + "objective/non_score_reward": -1.7162445783615112, + "objective/rlhf_reward": 13.9556303024292, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008753975853323936, + "policy/clipfrac_avg": 0.006253514904528856, + "policy/entropy_avg": 0.17108154296875, + "step": 729, + "val/clipfrac_avg": 0.03564208000898361, + "val/num_eos_tokens": 194, + "val/ratio": 0.9883392453193665, + "val/ratio_var": 5.439678716356866e-05 + }, + { + "episode": 11680, + "epoch": 0.5898989898989899, + "eps": 0, + "loss/policy_avg": 0.5118429064750671, + "loss/value_avg": 0.3809865713119507, + "lr": 3.9526037486409275e-06, + "objective/entropy": -211.5462646484375, + "objective/kl": 38.28752136230469, + "objective/non_score_reward": -1.9143760204315186, + "objective/rlhf_reward": 13.062187194824219, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008200224256142974, + "policy/clipfrac_avg": 0.009272422641515732, + "policy/entropy_avg": 0.15618228912353516, + "step": 730, + "val/clipfrac_avg": 0.13058164715766907, + "val/num_eos_tokens": 191, + "val/ratio": 0.9892928600311279, + "val/ratio_var": 5.910802065045573e-05 + }, + { + "episode": 11696, + "epoch": 0.5907070707070707, + "eps": 0, + "loss/policy_avg": -0.575583279132843, + "loss/value_avg": 0.6588011384010315, + "lr": 3.939546742922318e-06, + "objective/entropy": -155.975341796875, + "objective/kl": 33.536251068115234, + "objective/non_score_reward": -1.6768126487731934, + "objective/rlhf_reward": 14.198186874389648, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0006373925716616213, + "policy/clipfrac_avg": 0.011547036468982697, + "policy/entropy_avg": 0.15375041961669922, + "step": 731, + "val/clipfrac_avg": 0.1309666782617569, + "val/num_eos_tokens": 393, + "val/ratio": 0.9912899732589722, + "val/ratio_var": 7.494341116398573e-05 + }, + { + "episode": 11712, + "epoch": 0.5915151515151515, + "eps": 0, + "loss/policy_avg": 0.08975643664598465, + "loss/value_avg": 0.4319133758544922, + "lr": 3.926497305120943e-06, + "objective/entropy": -46.366912841796875, + "objective/kl": 38.771400451660156, + "objective/non_score_reward": -1.9385700225830078, + "objective/rlhf_reward": 14.631742477416992, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010276802349835634, + "policy/clipfrac_avg": 0.004419120494276285, + "policy/entropy_avg": 0.19952774047851562, + "step": 732, + "val/clipfrac_avg": 0.02601846680045128, + "val/num_eos_tokens": 121, + "val/ratio": 0.9856929779052734, + "val/ratio_var": 5.2173218136886135e-05 + }, + { + "episode": 11728, + "epoch": 0.5923232323232324, + "eps": 0, + "loss/policy_avg": -0.5671601295471191, + "loss/value_avg": 1.4009994268417358, + "lr": 3.913455528364027e-06, + "objective/entropy": -30.65240478515625, + "objective/kl": 34.644386291503906, + "objective/non_score_reward": -1.7322194576263428, + "objective/rlhf_reward": 16.127155303955078, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009229708230122924, + "policy/clipfrac_avg": 0.0053136469796299934, + "policy/entropy_avg": 0.1684565544128418, + "step": 733, + "val/clipfrac_avg": 0.16779303550720215, + "val/num_eos_tokens": 257, + "val/ratio": 0.98807692527771, + "val/ratio_var": 4.935808829031885e-05 + }, + { + "episode": 11744, + "epoch": 0.5931313131313132, + "eps": 0, + "loss/policy_avg": 0.1117314025759697, + "loss/value_avg": 0.6027213335037231, + "lr": 3.900421505724119e-06, + "objective/entropy": -115.830078125, + "objective/kl": 38.92997741699219, + "objective/non_score_reward": -1.9464988708496094, + "objective/rlhf_reward": 12.38162612915039, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006995117873884737, + "policy/clipfrac_avg": 0.0067988173104822636, + "policy/entropy_avg": 0.12786293029785156, + "step": 734, + "val/clipfrac_avg": 0.11902365833520889, + "val/num_eos_tokens": 256, + "val/ratio": 0.9913206696510315, + "val/ratio_var": 3.536290387273766e-05 + }, + { + "episode": 11760, + "epoch": 0.593939393939394, + "eps": 0, + "loss/policy_avg": 0.07707646489143372, + "loss/value_avg": 0.8557188510894775, + "lr": 3.887395330218429e-06, + "objective/entropy": -77.05795288085938, + "objective/kl": 34.79220199584961, + "objective/non_score_reward": -1.739610195159912, + "objective/rlhf_reward": 12.510390281677246, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009019911522045732, + "policy/clipfrac_avg": 0.00438590906560421, + "policy/entropy_avg": 0.18432998657226562, + "step": 735, + "val/clipfrac_avg": 0.20910227298736572, + "val/num_eos_tokens": 339, + "val/ratio": 0.9879709482192993, + "val/ratio_var": 7.31311592971906e-05 + }, + { + "episode": 11776, + "epoch": 0.5947474747474748, + "eps": 0, + "loss/policy_avg": 0.005743637681007385, + "loss/value_avg": 0.4823000729084015, + "lr": 3.8743770948081675e-06, + "objective/entropy": -189.58908081054688, + "objective/kl": 40.491024017333984, + "objective/non_score_reward": -2.0245513916015625, + "objective/rlhf_reward": 12.608261108398438, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010324950562790036, + "policy/clipfrac_avg": 0.008821255527436733, + "policy/entropy_avg": 0.1864786148071289, + "step": 736, + "val/clipfrac_avg": 0.03615584969520569, + "val/num_eos_tokens": 146, + "val/ratio": 0.9867680072784424, + "val/ratio_var": 0.00012203787628095597 + }, + { + "episode": 11792, + "epoch": 0.5955555555555555, + "eps": 0, + "loss/policy_avg": 0.11549818515777588, + "loss/value_avg": 0.4191131591796875, + "lr": 3.861366892397878e-06, + "objective/entropy": -53.57340621948242, + "objective/kl": 47.2681999206543, + "objective/non_score_reward": -2.363409996032715, + "objective/rlhf_reward": 11.238153457641602, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.0010416920995339751, + "policy/clipfrac_avg": 0.006328020244836807, + "policy/entropy_avg": 0.21573257446289062, + "step": 737, + "val/clipfrac_avg": 0.05454040318727493, + "val/num_eos_tokens": 145, + "val/ratio": 0.9863517880439758, + "val/ratio_var": 5.17384396516718e-05 + }, + { + "episode": 11808, + "epoch": 0.5963636363636363, + "eps": 0, + "loss/policy_avg": 0.09252296388149261, + "loss/value_avg": 0.24368464946746826, + "lr": 3.848364815834782e-06, + "objective/entropy": -9.551445007324219, + "objective/kl": 41.81903839111328, + "objective/non_score_reward": -2.090951919555664, + "objective/rlhf_reward": 15.362173080444336, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.001077502267435193, + "policy/clipfrac_avg": 0.007244379259645939, + "policy/entropy_avg": 0.208465576171875, + "step": 738, + "val/clipfrac_avg": 0.046727634966373444, + "val/num_eos_tokens": 70, + "val/ratio": 0.9859708547592163, + "val/ratio_var": 4.752089807880111e-05 + }, + { + "episode": 11824, + "epoch": 0.5971717171717171, + "eps": 0, + "loss/policy_avg": -0.47144564986228943, + "loss/value_avg": 0.9659825563430786, + "lr": 3.835370957908108e-06, + "objective/entropy": -109.4366226196289, + "objective/kl": 32.577239990234375, + "objective/non_score_reward": -1.6288621425628662, + "objective/rlhf_reward": 13.332075119018555, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0009311667527072132, + "policy/clipfrac_avg": 0.007746369112282991, + "policy/entropy_avg": 0.2024822235107422, + "step": 739, + "val/clipfrac_avg": 0.1542586386203766, + "val/num_eos_tokens": 140, + "val/ratio": 0.9888983964920044, + "val/ratio_var": 8.097994577838108e-05 + }, + { + "episode": 11840, + "epoch": 0.597979797979798, + "eps": 0, + "loss/policy_avg": 0.3934873938560486, + "loss/value_avg": 0.5160516500473022, + "lr": 3.822385411348429e-06, + "objective/entropy": -82.36312103271484, + "objective/kl": 38.616180419921875, + "objective/non_score_reward": -1.9308089017868042, + "objective/rlhf_reward": 12.725440979003906, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008991706417873502, + "policy/clipfrac_avg": 0.01843995228409767, + "policy/entropy_avg": 0.2245502471923828, + "step": 740, + "val/clipfrac_avg": 0.0067659346386790276, + "val/num_eos_tokens": 30, + "val/ratio": 0.988348662853241, + "val/ratio_var": 5.281575795379467e-05 + }, + { + "episode": 11856, + "epoch": 0.5987878787878788, + "eps": 0, + "loss/policy_avg": -0.06747608631849289, + "loss/value_avg": 0.3114325702190399, + "lr": 3.809408268827009e-06, + "objective/entropy": -89.22055053710938, + "objective/kl": 46.57951736450195, + "objective/non_score_reward": -2.3289759159088135, + "objective/rlhf_reward": 13.483524322509766, + "objective/scores": 15.8125, + "policy/approxkl_avg": 0.000996086047962308, + "policy/clipfrac_avg": 0.005896539893001318, + "policy/entropy_avg": 0.19659423828125, + "step": 741, + "val/clipfrac_avg": 0.0699368417263031, + "val/num_eos_tokens": 107, + "val/ratio": 0.9875314235687256, + "val/ratio_var": 4.793483094545081e-05 + }, + { + "episode": 11872, + "epoch": 0.5995959595959596, + "eps": 0, + "loss/policy_avg": -0.2737995684146881, + "loss/value_avg": 0.4772384762763977, + "lr": 3.7964396229551365e-06, + "objective/entropy": -104.62603759765625, + "objective/kl": 36.626800537109375, + "objective/non_score_reward": -1.831339955329895, + "objective/rlhf_reward": 14.778034210205078, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008760478813201189, + "policy/clipfrac_avg": 0.005249073728919029, + "policy/entropy_avg": 0.17531394958496094, + "step": 742, + "val/clipfrac_avg": 0.06328597664833069, + "val/num_eos_tokens": 91, + "val/ratio": 0.9885821342468262, + "val/ratio_var": 5.911157131777145e-05 + }, + { + "episode": 11888, + "epoch": 0.6004040404040404, + "eps": 0, + "loss/policy_avg": -0.11605305969715118, + "loss/value_avg": 0.30427810549736023, + "lr": 3.783479566283457e-06, + "objective/entropy": -123.75762176513672, + "objective/kl": 35.787086486816406, + "objective/non_score_reward": -1.7893544435501099, + "objective/rlhf_reward": 14.86689567565918, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0007770034717395902, + "policy/clipfrac_avg": 0.0056176697835326195, + "policy/entropy_avg": 0.152313232421875, + "step": 743, + "val/clipfrac_avg": 0.048395052552223206, + "val/num_eos_tokens": 386, + "val/ratio": 0.9905653595924377, + "val/ratio_var": 5.709798642783426e-05 + }, + { + "episode": 11904, + "epoch": 0.6012121212121212, + "eps": 0, + "loss/policy_avg": -0.02294028550386429, + "loss/value_avg": 0.7253668904304504, + "lr": 3.7705281913013286e-06, + "objective/entropy": -39.3416633605957, + "objective/kl": 43.74962615966797, + "objective/non_score_reward": -2.18748140335083, + "objective/rlhf_reward": 12.648456573486328, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010895703453570604, + "policy/clipfrac_avg": 0.0052008070051670074, + "policy/entropy_avg": 0.22774124145507812, + "step": 744, + "val/clipfrac_avg": 0.01829145848751068, + "val/num_eos_tokens": 71, + "val/ratio": 0.985338568687439, + "val/ratio_var": 6.19284255662933e-05 + }, + { + "episode": 11920, + "epoch": 0.602020202020202, + "eps": 0, + "loss/policy_avg": 0.1328388750553131, + "loss/value_avg": 1.3390746116638184, + "lr": 3.7575855904361447e-06, + "objective/entropy": -79.38550567626953, + "objective/kl": 33.780616760253906, + "objective/non_score_reward": -1.6890308856964111, + "objective/rlhf_reward": 14.178156852722168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0008661438478156924, + "policy/clipfrac_avg": 0.003558866214007139, + "policy/entropy_avg": 0.15964603424072266, + "step": 745, + "val/clipfrac_avg": 0.05471283197402954, + "val/num_eos_tokens": 81, + "val/ratio": 0.9897934198379517, + "val/ratio_var": 8.215933485189453e-05 + }, + { + "episode": 11936, + "epoch": 0.6028282828282828, + "eps": 0, + "loss/policy_avg": -0.29737263917922974, + "loss/value_avg": 0.6314270496368408, + "lr": 3.7446518560526903e-06, + "objective/entropy": -124.25365447998047, + "objective/kl": 42.908660888671875, + "objective/non_score_reward": -2.145432949066162, + "objective/rlhf_reward": 11.87019157409668, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009470224613323808, + "policy/clipfrac_avg": 0.006878838874399662, + "policy/entropy_avg": 0.1818103790283203, + "step": 746, + "val/clipfrac_avg": 0.0798804834485054, + "val/num_eos_tokens": 329, + "val/ratio": 0.9880188703536987, + "val/ratio_var": 8.261724724434316e-05 + }, + { + "episode": 11952, + "epoch": 0.6036363636363636, + "eps": 0, + "loss/policy_avg": 0.12454540282487869, + "loss/value_avg": 0.4487236738204956, + "lr": 3.731727080452464e-06, + "objective/entropy": -72.31340026855469, + "objective/kl": 43.510276794433594, + "objective/non_score_reward": -2.175513982772827, + "objective/rlhf_reward": 12.347923278808594, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009941391181200743, + "policy/clipfrac_avg": 0.00658874586224556, + "policy/entropy_avg": 0.16569900512695312, + "step": 747, + "val/clipfrac_avg": 0.1540203094482422, + "val/num_eos_tokens": 77, + "val/ratio": 0.9888790845870972, + "val/ratio_var": 2.4041321012191474e-05 + }, + { + "episode": 11968, + "epoch": 0.6044444444444445, + "eps": 0, + "loss/policy_avg": -0.1547861397266388, + "loss/value_avg": 0.49022629857063293, + "lr": 3.7188113558730403e-06, + "objective/entropy": -103.35200500488281, + "objective/kl": 35.28123474121094, + "objective/non_score_reward": -1.764061689376831, + "objective/rlhf_reward": 13.27500057220459, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0008191847009584308, + "policy/clipfrac_avg": 0.006343110464513302, + "policy/entropy_avg": 0.14904117584228516, + "step": 748, + "val/clipfrac_avg": 0.04430587589740753, + "val/num_eos_tokens": 124, + "val/ratio": 0.9908789396286011, + "val/ratio_var": 4.545380579656921e-05 + }, + { + "episode": 11984, + "epoch": 0.6052525252525253, + "eps": 0, + "loss/policy_avg": 0.11601711809635162, + "loss/value_avg": 0.6175556182861328, + "lr": 3.705904774487396e-06, + "objective/entropy": -111.66493225097656, + "objective/kl": 36.76349639892578, + "objective/non_score_reward": -1.838174819946289, + "objective/rlhf_reward": 13.693075180053711, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0008918631356209517, + "policy/clipfrac_avg": 0.010910662822425365, + "policy/entropy_avg": 0.16454505920410156, + "step": 749, + "val/clipfrac_avg": 0.06755826622247696, + "val/num_eos_tokens": 358, + "val/ratio": 0.9886500835418701, + "val/ratio_var": 6.255816697375849e-05 + }, + { + "episode": 12000, + "epoch": 0.6060606060606061, + "eps": 0, + "loss/policy_avg": -0.04627024754881859, + "loss/value_avg": 0.6224689483642578, + "lr": 3.6930074284032613e-06, + "objective/entropy": -63.2913703918457, + "objective/kl": 32.27269744873047, + "objective/non_score_reward": -1.6136349439620972, + "objective/rlhf_reward": 14.644177436828613, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009490256779827178, + "policy/clipfrac_avg": 0.0068245502188801765, + "policy/entropy_avg": 0.17229938507080078, + "step": 750, + "val/clipfrac_avg": 0.18630267679691315, + "val/num_eos_tokens": 11, + "val/ratio": 0.9878020882606506, + "val/ratio_var": 5.455148493638262e-05 + }, + { + "episode": 12016, + "epoch": 0.6068686868686869, + "eps": 0, + "loss/policy_avg": -0.17890788614749908, + "loss/value_avg": 0.7140947580337524, + "lr": 3.6801194096624515e-06, + "objective/entropy": -54.77946472167969, + "objective/kl": 34.44614028930664, + "objective/non_score_reward": -1.7223069667816162, + "objective/rlhf_reward": 14.894880294799805, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0011779924388974905, + "policy/clipfrac_avg": 0.024402525275945663, + "policy/entropy_avg": 0.2098255157470703, + "step": 751, + "val/clipfrac_avg": 0.04182715713977814, + "val/num_eos_tokens": 42, + "val/ratio": 0.9854939579963684, + "val/ratio_var": 0.00014133901277091354 + }, + { + "episode": 12032, + "epoch": 0.6076767676767677, + "eps": 0, + "loss/policy_avg": -0.0015007667243480682, + "loss/value_avg": 0.2544102668762207, + "lr": 3.6672408102402247e-06, + "objective/entropy": -70.776611328125, + "objective/kl": 37.67994689941406, + "objective/non_score_reward": -1.8839974403381348, + "objective/rlhf_reward": 12.108190536499023, + "objective/scores": 14.0, + "policy/approxkl_avg": 0.0009373503271490335, + "policy/clipfrac_avg": 0.005210112780332565, + "policy/entropy_avg": 0.18710041046142578, + "step": 752, + "val/clipfrac_avg": 0.021128622815012932, + "val/num_eos_tokens": 218, + "val/ratio": 0.9874061942100525, + "val/ratio_var": 5.212551332078874e-05 + }, + { + "episode": 12048, + "epoch": 0.6084848484848485, + "eps": 0, + "loss/policy_avg": 0.05351484939455986, + "loss/value_avg": 0.28849858045578003, + "lr": 3.654371722044616e-06, + "objective/entropy": -89.17417907714844, + "objective/kl": 44.5589714050293, + "objective/non_score_reward": -2.2279486656188965, + "objective/rlhf_reward": 12.178300857543945, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0009375615045428276, + "policy/clipfrac_avg": 0.005882620811462402, + "policy/entropy_avg": 0.18878555297851562, + "step": 753, + "val/clipfrac_avg": 0.021566759794950485, + "val/num_eos_tokens": 120, + "val/ratio": 0.9880841970443726, + "val/ratio_var": 5.727734605898149e-05 + }, + { + "episode": 12064, + "epoch": 0.6092929292929293, + "eps": 0, + "loss/policy_avg": -0.12424173951148987, + "loss/value_avg": 0.359388530254364, + "lr": 3.641512236915783e-06, + "objective/entropy": -114.3979263305664, + "objective/kl": 43.28697967529297, + "objective/non_score_reward": -2.16434907913208, + "objective/rlhf_reward": 14.226276397705078, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.000866133370436728, + "policy/clipfrac_avg": 0.0050315093249082565, + "policy/entropy_avg": 0.16837501525878906, + "step": 754, + "val/clipfrac_avg": 0.1426960825920105, + "val/num_eos_tokens": 391, + "val/ratio": 0.9894628524780273, + "val/ratio_var": 5.016153590986505e-05 + }, + { + "episode": 12080, + "epoch": 0.6101010101010101, + "eps": 0, + "loss/policy_avg": -0.2821425795555115, + "loss/value_avg": 0.38819342851638794, + "lr": 3.6286624466253496e-06, + "objective/entropy": -164.57290649414062, + "objective/kl": 38.83340072631836, + "objective/non_score_reward": -1.9416700601577759, + "objective/rlhf_reward": 14.245829582214355, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008140793070197105, + "policy/clipfrac_avg": 0.00435154652222991, + "policy/entropy_avg": 0.15729975700378418, + "step": 755, + "val/clipfrac_avg": 0.09525477886199951, + "val/num_eos_tokens": 336, + "val/ratio": 0.989446222782135, + "val/ratio_var": 5.3185020078672096e-05 + }, + { + "episode": 12096, + "epoch": 0.610909090909091, + "eps": 0, + "loss/policy_avg": 0.05094696208834648, + "loss/value_avg": 0.4260476529598236, + "lr": 3.6158224428757538e-06, + "objective/entropy": -61.176116943359375, + "objective/kl": 32.80990219116211, + "objective/non_score_reward": -1.6404950618743896, + "objective/rlhf_reward": 16.24231719970703, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009597806492820382, + "policy/clipfrac_avg": 0.0061972904950380325, + "policy/entropy_avg": 0.16566753387451172, + "step": 756, + "val/clipfrac_avg": 0.01815173029899597, + "val/num_eos_tokens": 163, + "val/ratio": 0.9894044399261475, + "val/ratio_var": 4.711053770733997e-05 + }, + { + "episode": 12112, + "epoch": 0.6117171717171718, + "eps": 0, + "loss/policy_avg": -0.040469832718372345, + "loss/value_avg": 0.3746616244316101, + "lr": 3.602992317299594e-06, + "objective/entropy": -52.30070877075195, + "objective/kl": 34.218990325927734, + "objective/non_score_reward": -1.7109496593475342, + "objective/rlhf_reward": 13.273425102233887, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009450222714804113, + "policy/clipfrac_avg": 0.0049097794108092785, + "policy/entropy_avg": 0.1805877685546875, + "step": 757, + "val/clipfrac_avg": 0.09231511503458023, + "val/num_eos_tokens": 105, + "val/ratio": 0.9877369403839111, + "val/ratio_var": 4.407649248605594e-05 + }, + { + "episode": 12128, + "epoch": 0.6125252525252525, + "eps": 0, + "loss/policy_avg": -0.0790981650352478, + "loss/value_avg": 0.289002388715744, + "lr": 3.5901721614589725e-06, + "objective/entropy": -33.08611297607422, + "objective/kl": 42.89472961425781, + "objective/non_score_reward": -2.1447365283966064, + "objective/rlhf_reward": 14.792763710021973, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0011628612410277128, + "policy/clipfrac_avg": 0.00601824838668108, + "policy/entropy_avg": 0.21850204467773438, + "step": 758, + "val/clipfrac_avg": 0.05997534096240997, + "val/num_eos_tokens": 208, + "val/ratio": 0.9858998656272888, + "val/ratio_var": 4.6025765186641365e-05 + }, + { + "episode": 12144, + "epoch": 0.6133333333333333, + "eps": 0, + "loss/policy_avg": 0.08223091810941696, + "loss/value_avg": 0.4521709978580475, + "lr": 3.5773620668448384e-06, + "objective/entropy": -57.002418518066406, + "objective/kl": 34.86107635498047, + "objective/non_score_reward": -1.743053674697876, + "objective/rlhf_reward": 12.421009063720703, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0009696290362626314, + "policy/clipfrac_avg": 0.005794525146484375, + "policy/entropy_avg": 0.1826324462890625, + "step": 759, + "val/clipfrac_avg": 0.03934604302048683, + "val/num_eos_tokens": 137, + "val/ratio": 0.9880379438400269, + "val/ratio_var": 4.820780668524094e-05 + }, + { + "episode": 12160, + "epoch": 0.6141414141414141, + "eps": 0, + "loss/policy_avg": 0.012903919443488121, + "loss/value_avg": 0.22128555178642273, + "lr": 3.564562124876344e-06, + "objective/entropy": -58.77183532714844, + "objective/kl": 40.15751647949219, + "objective/non_score_reward": -2.007875919342041, + "objective/rlhf_reward": 13.171812057495117, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0009755170904099941, + "policy/clipfrac_avg": 0.0075699929147958755, + "policy/entropy_avg": 0.1793661117553711, + "step": 760, + "val/clipfrac_avg": 0.007695037871599197, + "val/num_eos_tokens": 194, + "val/ratio": 0.9879107475280762, + "val/ratio_var": 9.122795017901808e-05 + }, + { + "episode": 12176, + "epoch": 0.6149494949494949, + "eps": 0, + "loss/policy_avg": 0.23269692063331604, + "loss/value_avg": 0.6034168004989624, + "lr": 3.5517724269001853e-06, + "objective/entropy": -92.11294555664062, + "objective/kl": 36.40907287597656, + "objective/non_score_reward": -1.8204538822174072, + "objective/rlhf_reward": 13.468608856201172, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009949133964255452, + "policy/clipfrac_avg": 0.012031201273202896, + "policy/entropy_avg": 0.19497203826904297, + "step": 761, + "val/clipfrac_avg": 0.03142271935939789, + "val/num_eos_tokens": 210, + "val/ratio": 0.9871051907539368, + "val/ratio_var": 7.474326412193477e-05 + }, + { + "episode": 12192, + "epoch": 0.6157575757575757, + "eps": 0, + "loss/policy_avg": -0.014487015083432198, + "loss/value_avg": 1.0809375047683716, + "lr": 3.538993064189954e-06, + "objective/entropy": -42.83490753173828, + "objective/kl": 32.94031524658203, + "objective/non_score_reward": -1.6470158100128174, + "objective/rlhf_reward": 13.567828178405762, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0011591427028179169, + "policy/clipfrac_avg": 0.004484353121370077, + "policy/entropy_avg": 0.21469688415527344, + "step": 762, + "val/clipfrac_avg": 0.1017560362815857, + "val/num_eos_tokens": 453, + "val/ratio": 0.9864501953125, + "val/ratio_var": 8.947903552325442e-05 + }, + { + "episode": 12208, + "epoch": 0.6165656565656565, + "eps": 0, + "loss/policy_avg": -0.0410991832613945, + "loss/value_avg": 1.3014378547668457, + "lr": 3.526224127945479e-06, + "objective/entropy": -71.51974487304688, + "objective/kl": 31.28769302368164, + "objective/non_score_reward": -1.5643846988677979, + "objective/rlhf_reward": 15.216865539550781, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0007811100222170353, + "policy/clipfrac_avg": 0.0053521376103162766, + "policy/entropy_avg": 0.16764509677886963, + "step": 763, + "val/clipfrac_avg": 0.059704042971134186, + "val/num_eos_tokens": 203, + "val/ratio": 0.9894566535949707, + "val/ratio_var": 6.841208232799545e-05 + }, + { + "episode": 12224, + "epoch": 0.6173737373737374, + "eps": 0, + "loss/policy_avg": 0.112126424908638, + "loss/value_avg": 0.725554347038269, + "lr": 3.5134657092921877e-06, + "objective/entropy": -99.9854736328125, + "objective/kl": 41.09782028198242, + "objective/non_score_reward": -2.0548908710479736, + "objective/rlhf_reward": 10.624795913696289, + "objective/scores": 12.6875, + "policy/approxkl_avg": 0.0009035035036504269, + "policy/clipfrac_avg": 0.010427422821521759, + "policy/entropy_avg": 0.17998504638671875, + "step": 764, + "val/clipfrac_avg": 0.12764863669872284, + "val/num_eos_tokens": 226, + "val/ratio": 0.988477349281311, + "val/ratio_var": 8.909796451916918e-05 + }, + { + "episode": 12240, + "epoch": 0.6181818181818182, + "eps": 0, + "loss/policy_avg": 0.37758830189704895, + "loss/value_avg": 0.7465800046920776, + "lr": 3.500717899280442e-06, + "objective/entropy": -135.23390197753906, + "objective/kl": 41.466583251953125, + "objective/non_score_reward": -2.073329210281372, + "objective/rlhf_reward": 12.94229507446289, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0009621024364605546, + "policy/clipfrac_avg": 0.00628144945949316, + "policy/entropy_avg": 0.17942047119140625, + "step": 765, + "val/clipfrac_avg": 0.10096652805805206, + "val/num_eos_tokens": 233, + "val/ratio": 0.9877568483352661, + "val/ratio_var": 7.067532715154812e-05 + }, + { + "episode": 12256, + "epoch": 0.618989898989899, + "eps": 0, + "loss/policy_avg": -0.10782650113105774, + "loss/value_avg": 0.4996359944343567, + "lr": 3.4879807888849026e-06, + "objective/entropy": -86.19502258300781, + "objective/kl": 39.61420440673828, + "objective/non_score_reward": -1.9807100296020508, + "objective/rlhf_reward": 13.706789016723633, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009516358841210604, + "policy/clipfrac_avg": 0.01008635200560093, + "policy/entropy_avg": 0.17633438110351562, + "step": 766, + "val/clipfrac_avg": 0.05575938522815704, + "val/num_eos_tokens": 426, + "val/ratio": 0.9884860515594482, + "val/ratio_var": 5.413329563452862e-05 + }, + { + "episode": 12272, + "epoch": 0.6197979797979798, + "eps": 0, + "loss/policy_avg": 0.0611935630440712, + "loss/value_avg": 0.40072089433670044, + "lr": 3.475254469003865e-06, + "objective/entropy": -78.26728057861328, + "objective/kl": 38.48719787597656, + "objective/non_score_reward": -1.9243600368499756, + "objective/rlhf_reward": 11.661577224731445, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009466722258366644, + "policy/clipfrac_avg": 0.0042708818800747395, + "policy/entropy_avg": 0.17498779296875, + "step": 767, + "val/clipfrac_avg": 0.1151452288031578, + "val/num_eos_tokens": 415, + "val/ratio": 0.9874659776687622, + "val/ratio_var": 7.443072536261752e-05 + }, + { + "episode": 12288, + "epoch": 0.6206060606060606, + "eps": 0, + "loss/policy_avg": -0.07818017899990082, + "loss/value_avg": 0.588293731212616, + "lr": 3.4625390304586224e-06, + "objective/entropy": -45.31693649291992, + "objective/kl": 41.06260681152344, + "objective/non_score_reward": -2.0531303882598877, + "objective/rlhf_reward": 13.860933303833008, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0011651598615571856, + "policy/clipfrac_avg": 0.0047182347625494, + "policy/entropy_avg": 0.2090606689453125, + "step": 768, + "val/clipfrac_avg": 0.16338574886322021, + "val/num_eos_tokens": 9, + "val/ratio": 0.986497700214386, + "val/ratio_var": 8.175932453013957e-05 + }, + { + "episode": 12304, + "epoch": 0.6214141414141414, + "eps": 0, + "loss/policy_avg": -0.3265213966369629, + "loss/value_avg": 0.5878744125366211, + "lr": 3.4498345639928167e-06, + "objective/entropy": -103.96510314941406, + "objective/kl": 35.794551849365234, + "objective/non_score_reward": -1.7897276878356934, + "objective/rlhf_reward": 13.507146835327148, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0010084104724228382, + "policy/clipfrac_avg": 0.0070042358711361885, + "policy/entropy_avg": 0.19075870513916016, + "step": 769, + "val/clipfrac_avg": 0.12878292798995972, + "val/num_eos_tokens": 8, + "val/ratio": 0.9861502051353455, + "val/ratio_var": 8.973718649940565e-05 + }, + { + "episode": 12320, + "epoch": 0.6222222222222222, + "eps": 0, + "loss/policy_avg": -0.24864813685417175, + "loss/value_avg": 0.703667402267456, + "lr": 3.4371411602717785e-06, + "objective/entropy": -92.59934997558594, + "objective/kl": 31.87890625, + "objective/non_score_reward": -1.5939452648162842, + "objective/rlhf_reward": 14.429492950439453, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0007836055592633784, + "policy/clipfrac_avg": 0.004061062820255756, + "policy/entropy_avg": 0.13613224029541016, + "step": 770, + "val/clipfrac_avg": 0.14356926083564758, + "val/num_eos_tokens": 22, + "val/ratio": 0.9909623265266418, + "val/ratio_var": 6.252773891901597e-05 + }, + { + "episode": 12336, + "epoch": 0.623030303030303, + "eps": 0, + "loss/policy_avg": -0.11106158792972565, + "loss/value_avg": 0.7719787359237671, + "lr": 3.424458909881897e-06, + "objective/entropy": -201.825927734375, + "objective/kl": 32.05082321166992, + "objective/non_score_reward": -1.602541208267212, + "objective/rlhf_reward": 13.842771530151367, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0009224842651747167, + "policy/clipfrac_avg": 0.004090993665158749, + "policy/entropy_avg": 0.19452500343322754, + "step": 771, + "val/clipfrac_avg": 0.06381858140230179, + "val/num_eos_tokens": 137, + "val/ratio": 0.9873221516609192, + "val/ratio_var": 0.00010800129530252889 + }, + { + "episode": 12352, + "epoch": 0.6238383838383839, + "eps": 0, + "loss/policy_avg": -0.3147702217102051, + "loss/value_avg": 1.0198739767074585, + "lr": 3.411787903329963e-06, + "objective/entropy": -97.80130767822266, + "objective/kl": 28.94087791442871, + "objective/non_score_reward": -1.4470438957214355, + "objective/rlhf_reward": 13.572486877441406, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0008233180269598961, + "policy/clipfrac_avg": 0.007523335050791502, + "policy/entropy_avg": 0.17452716827392578, + "step": 772, + "val/clipfrac_avg": 0.021688329055905342, + "val/num_eos_tokens": 116, + "val/ratio": 0.9885031580924988, + "val/ratio_var": 6.793635111534968e-05 + }, + { + "episode": 12368, + "epoch": 0.6246464646464647, + "eps": 0, + "loss/policy_avg": -0.019919998943805695, + "loss/value_avg": 0.31691867113113403, + "lr": 3.3991282310425276e-06, + "objective/entropy": -61.794166564941406, + "objective/kl": 33.001129150390625, + "objective/non_score_reward": -1.6500564813613892, + "objective/rlhf_reward": 12.943693161010742, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009417567634955049, + "policy/clipfrac_avg": 0.005282199941575527, + "policy/entropy_avg": 0.1732311248779297, + "step": 773, + "val/clipfrac_avg": 0.05137361213564873, + "val/num_eos_tokens": 13, + "val/ratio": 0.9889352321624756, + "val/ratio_var": 3.9570189983351156e-05 + }, + { + "episode": 12384, + "epoch": 0.6254545454545455, + "eps": 0, + "loss/policy_avg": 0.010051839053630829, + "loss/value_avg": 0.5414019227027893, + "lr": 3.386479983365251e-06, + "objective/entropy": -73.91504669189453, + "objective/kl": 29.43255615234375, + "objective/non_score_reward": -1.4716278314590454, + "objective/rlhf_reward": 13.293996810913086, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.0010180522222071886, + "policy/clipfrac_avg": 0.0050889430567622185, + "policy/entropy_avg": 0.2153911590576172, + "step": 774, + "val/clipfrac_avg": 0.041913747787475586, + "val/num_eos_tokens": 214, + "val/ratio": 0.986315906047821, + "val/ratio_var": 8.102346328087151e-05 + }, + { + "episode": 12400, + "epoch": 0.6262626262626263, + "eps": 0, + "loss/policy_avg": -0.040474407374858856, + "loss/value_avg": 0.30427631735801697, + "lr": 3.3738432505622653e-06, + "objective/entropy": -75.78406524658203, + "objective/kl": 31.846160888671875, + "objective/non_score_reward": -1.5923081636428833, + "objective/rlhf_reward": 11.946754455566406, + "objective/scores": 13.5625, + "policy/approxkl_avg": 0.0009044273756444454, + "policy/clipfrac_avg": 0.005002772435545921, + "policy/entropy_avg": 0.18775272369384766, + "step": 775, + "val/clipfrac_avg": 0.04153668135404587, + "val/num_eos_tokens": 243, + "val/ratio": 0.9877529144287109, + "val/ratio_var": 5.462392800836824e-05 + }, + { + "episode": 12416, + "epoch": 0.6270707070707071, + "eps": 0, + "loss/policy_avg": -0.25257226824760437, + "loss/value_avg": 0.47636741399765015, + "lr": 3.3612181228155237e-06, + "objective/entropy": -97.84140014648438, + "objective/kl": 35.974266052246094, + "objective/non_score_reward": -1.798713207244873, + "objective/rlhf_reward": 13.685661315917969, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009363207500427961, + "policy/clipfrac_avg": 0.0035052683670073748, + "policy/entropy_avg": 0.18935298919677734, + "step": 776, + "val/clipfrac_avg": 0.07260937243700027, + "val/num_eos_tokens": 26, + "val/ratio": 0.9877586364746094, + "val/ratio_var": 0.00010292992374161258 + }, + { + "episode": 12432, + "epoch": 0.6278787878787879, + "eps": 0, + "loss/policy_avg": 0.41764986515045166, + "loss/value_avg": 0.8629061579704285, + "lr": 3.3486046902241663e-06, + "objective/entropy": -40.24072265625, + "objective/kl": 29.197486877441406, + "objective/non_score_reward": -1.4598743915557861, + "objective/rlhf_reward": 12.883874893188477, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010831760009750724, + "policy/clipfrac_avg": 0.02700810506939888, + "policy/entropy_avg": 0.16212844848632812, + "step": 777, + "val/clipfrac_avg": 0.08423963189125061, + "val/num_eos_tokens": 192, + "val/ratio": 0.9885735511779785, + "val/ratio_var": 8.724266081117094e-05 + }, + { + "episode": 12448, + "epoch": 0.6286868686868687, + "eps": 0, + "loss/policy_avg": 0.2242026925086975, + "loss/value_avg": 0.9421164989471436, + "lr": 3.336003042803861e-06, + "objective/entropy": -69.478271484375, + "objective/kl": 33.57006072998047, + "objective/non_score_reward": -1.6785030364990234, + "objective/rlhf_reward": 14.055871963500977, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008899248205125332, + "policy/clipfrac_avg": 0.006825867109000683, + "policy/entropy_avg": 0.14767837524414062, + "step": 778, + "val/clipfrac_avg": 0.1501787304878235, + "val/num_eos_tokens": 109, + "val/ratio": 0.9905418753623962, + "val/ratio_var": 4.142183388466947e-05 + }, + { + "episode": 12464, + "epoch": 0.6294949494949494, + "eps": 0, + "loss/policy_avg": 0.03614666312932968, + "loss/value_avg": 0.7387021780014038, + "lr": 3.3234132704861786e-06, + "objective/entropy": -182.4083251953125, + "objective/kl": 31.944690704345703, + "objective/non_score_reward": -1.5972344875335693, + "objective/rlhf_reward": 12.043390274047852, + "objective/scores": 13.625, + "policy/approxkl_avg": 0.000871892087161541, + "policy/clipfrac_avg": 0.004613575059920549, + "policy/entropy_avg": 0.1733226776123047, + "step": 779, + "val/clipfrac_avg": 0.1070820540189743, + "val/num_eos_tokens": 28, + "val/ratio": 0.9880519509315491, + "val/ratio_var": 9.137512097368017e-05 + }, + { + "episode": 12480, + "epoch": 0.6303030303030303, + "eps": 0, + "loss/policy_avg": 0.491243839263916, + "loss/value_avg": 1.7449349164962769, + "lr": 3.310835463117942e-06, + "objective/entropy": -80.43102264404297, + "objective/kl": 22.97904396057129, + "objective/non_score_reward": -1.1489522457122803, + "objective/rlhf_reward": 13.28073501586914, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.000753985601477325, + "policy/clipfrac_avg": 0.013304230757057667, + "policy/entropy_avg": 0.16235482692718506, + "step": 780, + "val/clipfrac_avg": 0.12828537821769714, + "val/num_eos_tokens": 261, + "val/ratio": 0.991666316986084, + "val/ratio_var": 4.57862151961308e-05 + }, + { + "episode": 12496, + "epoch": 0.6311111111111111, + "eps": 0, + "loss/policy_avg": -0.02106083557009697, + "loss/value_avg": 0.26465463638305664, + "lr": 3.2982697104605855e-06, + "objective/entropy": -96.58609008789062, + "objective/kl": 37.96983337402344, + "objective/non_score_reward": -1.898491621017456, + "objective/rlhf_reward": 12.742133140563965, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0008919077808968723, + "policy/clipfrac_avg": 0.004138793330639601, + "policy/entropy_avg": 0.176055908203125, + "step": 781, + "val/clipfrac_avg": 0.022776879370212555, + "val/num_eos_tokens": 232, + "val/ratio": 0.9880313873291016, + "val/ratio_var": 5.30094766872935e-05 + }, + { + "episode": 12512, + "epoch": 0.6319191919191919, + "eps": 0, + "loss/policy_avg": -0.21595685184001923, + "loss/value_avg": 0.1668311059474945, + "lr": 3.2857161021895125e-06, + "objective/entropy": -113.97618103027344, + "objective/kl": 30.898021697998047, + "objective/non_score_reward": -1.5449011325836182, + "objective/rlhf_reward": 15.892599105834961, + "objective/scores": 17.5, + "policy/approxkl_avg": 0.0008122409926727414, + "policy/clipfrac_avg": 0.007180417887866497, + "policy/entropy_avg": 0.15228557586669922, + "step": 782, + "val/clipfrac_avg": 0.04710119590163231, + "val/num_eos_tokens": 88, + "val/ratio": 0.9905120134353638, + "val/ratio_var": 4.6862616727594286e-05 + }, + { + "episode": 12528, + "epoch": 0.6327272727272727, + "eps": 0, + "loss/policy_avg": -0.02830284647643566, + "loss/value_avg": 0.470698744058609, + "lr": 3.273174727893463e-06, + "objective/entropy": -119.84375762939453, + "objective/kl": 35.46149444580078, + "objective/non_score_reward": -1.773074746131897, + "objective/rlhf_reward": 13.00036334991455, + "objective/scores": 14.75, + "policy/approxkl_avg": 0.000846280949190259, + "policy/clipfrac_avg": 0.006307992152869701, + "policy/entropy_avg": 0.15985870361328125, + "step": 783, + "val/clipfrac_avg": 0.09303553402423859, + "val/num_eos_tokens": 242, + "val/ratio": 0.9886760711669922, + "val/ratio_var": 8.341170905623585e-05 + }, + { + "episode": 12544, + "epoch": 0.6335353535353535, + "eps": 0, + "loss/policy_avg": -0.03483296558260918, + "loss/value_avg": 0.2278536707162857, + "lr": 3.2606456770738636e-06, + "objective/entropy": -32.83402633666992, + "objective/kl": 35.053001403808594, + "objective/non_score_reward": -1.7526500225067139, + "objective/rlhf_reward": 13.708287239074707, + "objective/scores": 15.4375, + "policy/approxkl_avg": 0.0010523913661018014, + "policy/clipfrac_avg": 0.005916148889809847, + "policy/entropy_avg": 0.2171039581298828, + "step": 784, + "val/clipfrac_avg": 0.007893375121057034, + "val/num_eos_tokens": 463, + "val/ratio": 0.9859672784805298, + "val/ratio_var": 5.139111090102233e-05 + }, + { + "episode": 12560, + "epoch": 0.6343434343434343, + "eps": 0, + "loss/policy_avg": 0.17171788215637207, + "loss/value_avg": 0.5937256217002869, + "lr": 3.2481290391442034e-06, + "objective/entropy": -109.50154113769531, + "objective/kl": 41.38114929199219, + "objective/non_score_reward": -2.0690574645996094, + "objective/rlhf_reward": 13.82156753540039, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009320294484496117, + "policy/clipfrac_avg": 0.005337849725037813, + "policy/entropy_avg": 0.1828467845916748, + "step": 785, + "val/clipfrac_avg": 0.0434221550822258, + "val/num_eos_tokens": 252, + "val/ratio": 0.9880971908569336, + "val/ratio_var": 7.803420157870278e-05 + }, + { + "episode": 12576, + "epoch": 0.6351515151515151, + "eps": 0, + "loss/policy_avg": -0.12326152622699738, + "loss/value_avg": 0.22453081607818604, + "lr": 3.235624903429374e-06, + "objective/entropy": -80.62982940673828, + "objective/kl": 39.33965301513672, + "objective/non_score_reward": -1.9669828414916992, + "objective/rlhf_reward": 14.3767671585083, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009726607240736485, + "policy/clipfrac_avg": 0.005861825309693813, + "policy/entropy_avg": 0.18674850463867188, + "step": 786, + "val/clipfrac_avg": 0.04197417199611664, + "val/num_eos_tokens": 55, + "val/ratio": 0.9867269396781921, + "val/ratio_var": 3.261497477069497e-05 + }, + { + "episode": 12592, + "epoch": 0.635959595959596, + "eps": 0, + "loss/policy_avg": 0.02891068160533905, + "loss/value_avg": 0.2414318025112152, + "lr": 3.2231333591650567e-06, + "objective/entropy": -64.31414031982422, + "objective/kl": 46.58558654785156, + "objective/non_score_reward": -2.3292794227600098, + "objective/rlhf_reward": 14.576970100402832, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009546708315610886, + "policy/clipfrac_avg": 0.00552853848785162, + "policy/entropy_avg": 0.189300537109375, + "step": 787, + "val/clipfrac_avg": 0.0013757861452177167, + "val/num_eos_tokens": 363, + "val/ratio": 0.987205982208252, + "val/ratio_var": 3.5099023079965264e-05 + }, + { + "episode": 12608, + "epoch": 0.6367676767676768, + "eps": 0, + "loss/policy_avg": -0.19538097083568573, + "loss/value_avg": 0.7160595655441284, + "lr": 3.210654495497068e-06, + "objective/entropy": -57.81846237182617, + "objective/kl": 42.85956573486328, + "objective/non_score_reward": -2.1429784297943115, + "objective/rlhf_reward": 12.14999008178711, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0011135776294395328, + "policy/clipfrac_avg": 0.009378955699503422, + "policy/entropy_avg": 0.2034454345703125, + "step": 788, + "val/clipfrac_avg": 0.1690627932548523, + "val/num_eos_tokens": 481, + "val/ratio": 0.9861621856689453, + "val/ratio_var": 9.24188134376891e-05 + }, + { + "episode": 12624, + "epoch": 0.6375757575757576, + "eps": 0, + "loss/policy_avg": 0.048018310219049454, + "loss/value_avg": 0.4312187135219574, + "lr": 3.198188401480734e-06, + "objective/entropy": -76.67124938964844, + "objective/kl": 36.849369049072266, + "objective/non_score_reward": -1.8424686193466187, + "objective/rlhf_reward": 12.25128173828125, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.0009952555410563946, + "policy/clipfrac_avg": 0.004150367341935635, + "policy/entropy_avg": 0.18071389198303223, + "step": 789, + "val/clipfrac_avg": 0.08804323524236679, + "val/num_eos_tokens": 357, + "val/ratio": 0.9867403507232666, + "val/ratio_var": 0.00013288675108924508 + }, + { + "episode": 12640, + "epoch": 0.6383838383838384, + "eps": 0, + "loss/policy_avg": 0.18971526622772217, + "loss/value_avg": 0.6626036167144775, + "lr": 3.185735166080244e-06, + "objective/entropy": -126.28926086425781, + "objective/kl": 33.96638488769531, + "objective/non_score_reward": -1.6983189582824707, + "objective/rlhf_reward": 12.973556518554688, + "objective/scores": 14.6875, + "policy/approxkl_avg": 0.0007302669691853225, + "policy/clipfrac_avg": 0.008397979661822319, + "policy/entropy_avg": 0.14786386489868164, + "step": 790, + "val/clipfrac_avg": 0.06916166841983795, + "val/num_eos_tokens": 600, + "val/ratio": 0.9910507798194885, + "val/ratio_var": 5.283964492264204e-05 + }, + { + "episode": 12656, + "epoch": 0.6391919191919192, + "eps": 0, + "loss/policy_avg": -0.4279332160949707, + "loss/value_avg": 0.5117957592010498, + "lr": 3.173294878168025e-06, + "objective/entropy": -75.64164733886719, + "objective/kl": 40.16947555541992, + "objective/non_score_reward": -2.0084738731384277, + "objective/rlhf_reward": 12.86652660369873, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008897786610759795, + "policy/clipfrac_avg": 0.004540015943348408, + "policy/entropy_avg": 0.18616324663162231, + "step": 791, + "val/clipfrac_avg": 0.0338541679084301, + "val/num_eos_tokens": 224, + "val/ratio": 0.9873502254486084, + "val/ratio_var": 6.438903801608831e-05 + }, + { + "episode": 12672, + "epoch": 0.64, + "eps": 0, + "loss/policy_avg": -0.30223172903060913, + "loss/value_avg": 0.7414761185646057, + "lr": 3.160867626524107e-06, + "objective/entropy": -36.1086540222168, + "objective/kl": 35.08636474609375, + "objective/non_score_reward": -1.7543182373046875, + "objective/rlhf_reward": 14.995681762695312, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0010164354462176561, + "policy/clipfrac_avg": 0.0033426620066165924, + "policy/entropy_avg": 0.20384597778320312, + "step": 792, + "val/clipfrac_avg": 0.08375494927167892, + "val/num_eos_tokens": 121, + "val/ratio": 0.9862762093544006, + "val/ratio_var": 8.216432615881786e-05 + }, + { + "episode": 12688, + "epoch": 0.6408080808080808, + "eps": 0, + "loss/policy_avg": 0.11556120216846466, + "loss/value_avg": 0.44696569442749023, + "lr": 3.1484534998354832e-06, + "objective/entropy": -52.89218521118164, + "objective/kl": 33.01978302001953, + "objective/non_score_reward": -1.650989294052124, + "objective/rlhf_reward": 12.270885467529297, + "objective/scores": 13.9375, + "policy/approxkl_avg": 0.0009637182811275125, + "policy/clipfrac_avg": 0.006595691200345755, + "policy/entropy_avg": 0.18551349639892578, + "step": 793, + "val/clipfrac_avg": 0.0313958078622818, + "val/num_eos_tokens": 483, + "val/ratio": 0.987555742263794, + "val/ratio_var": 4.9779242544900626e-05 + }, + { + "episode": 12704, + "epoch": 0.6416161616161616, + "eps": 0, + "loss/policy_avg": -0.11985203623771667, + "loss/value_avg": 0.5595318078994751, + "lr": 3.136052586695481e-06, + "objective/entropy": -44.2855110168457, + "objective/kl": 38.74113845825195, + "objective/non_score_reward": -1.9370570182800293, + "objective/rlhf_reward": 13.531692504882812, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009892593370750546, + "policy/clipfrac_avg": 0.006002710200846195, + "policy/entropy_avg": 0.20067405700683594, + "step": 794, + "val/clipfrac_avg": 0.051367055624723434, + "val/num_eos_tokens": 146, + "val/ratio": 0.98707115650177, + "val/ratio_var": 6.210069841472432e-05 + }, + { + "episode": 12720, + "epoch": 0.6424242424242425, + "eps": 0, + "loss/policy_avg": 0.06161116436123848, + "loss/value_avg": 0.20387178659439087, + "lr": 3.12366497560313e-06, + "objective/entropy": -47.063812255859375, + "objective/kl": 36.60810852050781, + "objective/non_score_reward": -1.8304054737091064, + "objective/rlhf_reward": 13.661782264709473, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000998336123302579, + "policy/clipfrac_avg": 0.004183639772236347, + "policy/entropy_avg": 0.19140625, + "step": 795, + "val/clipfrac_avg": 0.08834637701511383, + "val/num_eos_tokens": 342, + "val/ratio": 0.9874628782272339, + "val/ratio_var": 3.3792719477787614e-05 + }, + { + "episode": 12736, + "epoch": 0.6432323232323233, + "eps": 0, + "loss/policy_avg": 0.02652081847190857, + "loss/value_avg": 0.4432865381240845, + "lr": 3.1112907549625304e-06, + "objective/entropy": -104.79203796386719, + "objective/kl": 35.12114715576172, + "objective/non_score_reward": -1.7560575008392334, + "objective/rlhf_reward": 13.486129760742188, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0008146626059897244, + "policy/clipfrac_avg": 0.0056906514801084995, + "policy/entropy_avg": 0.1502237319946289, + "step": 796, + "val/clipfrac_avg": 0.061473242938518524, + "val/num_eos_tokens": 748, + "val/ratio": 0.9898946285247803, + "val/ratio_var": 4.619114042725414e-05 + }, + { + "episode": 12752, + "epoch": 0.6440404040404041, + "eps": 0, + "loss/policy_avg": 0.06834741681814194, + "loss/value_avg": 0.5731651782989502, + "lr": 3.098930013082223e-06, + "objective/entropy": -88.31727600097656, + "objective/kl": 38.224117279052734, + "objective/non_score_reward": -1.9112060070037842, + "objective/rlhf_reward": 14.698169708251953, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0008200570591725409, + "policy/clipfrac_avg": 0.007693252060562372, + "policy/entropy_avg": 0.1607208251953125, + "step": 797, + "val/clipfrac_avg": 0.11237858235836029, + "val/num_eos_tokens": 240, + "val/ratio": 0.9905504584312439, + "val/ratio_var": 4.706854815594852e-05 + }, + { + "episode": 12768, + "epoch": 0.6448484848484849, + "eps": 0, + "loss/policy_avg": -0.09761235117912292, + "loss/value_avg": 0.7088909149169922, + "lr": 3.0865828381745515e-06, + "objective/entropy": -124.7913589477539, + "objective/kl": 29.702392578125, + "objective/non_score_reward": -1.4851195812225342, + "objective/rlhf_reward": 13.68675422668457, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008782441727817059, + "policy/clipfrac_avg": 0.00598434591665864, + "policy/entropy_avg": 0.16924285888671875, + "step": 798, + "val/clipfrac_avg": 0.030641667544841766, + "val/num_eos_tokens": 454, + "val/ratio": 0.9882635474205017, + "val/ratio_var": 7.515518518630415e-05 + }, + { + "episode": 12784, + "epoch": 0.6456565656565657, + "eps": 0, + "loss/policy_avg": 0.14146611094474792, + "loss/value_avg": 0.7321012616157532, + "lr": 3.074249318355046e-06, + "objective/entropy": -101.5077133178711, + "objective/kl": 25.74894142150879, + "objective/non_score_reward": -1.2874470949172974, + "objective/rlhf_reward": 13.735990524291992, + "objective/scores": 15.0, + "policy/approxkl_avg": 0.0007512210868299007, + "policy/clipfrac_avg": 0.011982694268226624, + "policy/entropy_avg": 0.15892601013183594, + "step": 799, + "val/clipfrac_avg": 0.012677992694079876, + "val/num_eos_tokens": 12, + "val/ratio": 0.9910134077072144, + "val/ratio_var": 6.09022390563041e-05 + }, + { + "episode": 12800, + "epoch": 0.6464646464646465, + "eps": 0, + "loss/policy_avg": -0.042851828038692474, + "loss/value_avg": 0.3452759385108948, + "lr": 3.0619295416417856e-06, + "objective/entropy": -68.88214874267578, + "objective/kl": 34.023399353027344, + "objective/non_score_reward": -1.7011699676513672, + "objective/rlhf_reward": 12.494142532348633, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0010239251423627138, + "policy/clipfrac_avg": 0.010894561186432838, + "policy/entropy_avg": 0.17738628387451172, + "step": 800, + "val/clipfrac_avg": 0.026722928509116173, + "val/num_eos_tokens": 67, + "val/ratio": 0.9888595342636108, + "val/ratio_var": 6.888628558954224e-05 + }, + { + "episode": 12816, + "epoch": 0.6472727272727272, + "eps": 0, + "loss/policy_avg": -0.05754968896508217, + "loss/value_avg": 0.402332067489624, + "lr": 3.049623595954766e-06, + "objective/entropy": -89.62500762939453, + "objective/kl": 32.48891067504883, + "objective/non_score_reward": -1.6244455575942993, + "objective/rlhf_reward": 14.227116584777832, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0009385538287460804, + "policy/clipfrac_avg": 0.004920909181237221, + "policy/entropy_avg": 0.17440450191497803, + "step": 801, + "val/clipfrac_avg": 0.03176930546760559, + "val/num_eos_tokens": 9, + "val/ratio": 0.9880223274230957, + "val/ratio_var": 5.388163845054805e-05 + }, + { + "episode": 12832, + "epoch": 0.648080808080808, + "eps": 0, + "loss/policy_avg": 0.2483101636171341, + "loss/value_avg": 0.23584425449371338, + "lr": 3.0373315691152856e-06, + "objective/entropy": -89.31193542480469, + "objective/kl": 40.699127197265625, + "objective/non_score_reward": -2.034956455230713, + "objective/rlhf_reward": 12.160355567932129, + "objective/scores": 14.1875, + "policy/approxkl_avg": 0.0008679171442054212, + "policy/clipfrac_avg": 0.004822667222470045, + "policy/entropy_avg": 0.1660480499267578, + "step": 802, + "val/clipfrac_avg": 0.05813268944621086, + "val/num_eos_tokens": 149, + "val/ratio": 0.9890165328979492, + "val/ratio_var": 5.875104761798866e-05 + }, + { + "episode": 12848, + "epoch": 0.6488888888888888, + "eps": 0, + "loss/policy_avg": -0.10593897104263306, + "loss/value_avg": 0.5456434488296509, + "lr": 3.0250535488453077e-06, + "objective/entropy": -148.4324188232422, + "objective/kl": 38.51166915893555, + "objective/non_score_reward": -1.9255836009979248, + "objective/rlhf_reward": 11.918167114257812, + "objective/scores": 13.875, + "policy/approxkl_avg": 0.0007962615345604718, + "policy/clipfrac_avg": 0.0040261996909976006, + "policy/entropy_avg": 0.16094493865966797, + "step": 803, + "val/clipfrac_avg": 0.04293014109134674, + "val/num_eos_tokens": 331, + "val/ratio": 0.9893560409545898, + "val/ratio_var": 6.857667176518589e-05 + }, + { + "episode": 12864, + "epoch": 0.6496969696969697, + "eps": 0, + "loss/policy_avg": 0.06602153182029724, + "loss/value_avg": 0.2823004722595215, + "lr": 3.0127896227668367e-06, + "objective/entropy": -115.89801788330078, + "objective/kl": 40.921504974365234, + "objective/non_score_reward": -2.0460751056671143, + "objective/rlhf_reward": 12.867987632751465, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008664772612974048, + "policy/clipfrac_avg": 0.00966357346624136, + "policy/entropy_avg": 0.151214599609375, + "step": 804, + "val/clipfrac_avg": 0.03888742998242378, + "val/num_eos_tokens": 224, + "val/ratio": 0.9900601506233215, + "val/ratio_var": 5.00909227412194e-05 + }, + { + "episode": 12880, + "epoch": 0.6505050505050505, + "eps": 0, + "loss/policy_avg": -0.39892637729644775, + "loss/value_avg": 0.4240272641181946, + "lr": 3.000539878401296e-06, + "objective/entropy": -77.75199890136719, + "objective/kl": 35.056907653808594, + "objective/non_score_reward": -1.7528454065322876, + "objective/rlhf_reward": 14.00496768951416, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.001023960066959262, + "policy/clipfrac_avg": 0.004629913717508316, + "policy/entropy_avg": 0.20476627349853516, + "step": 805, + "val/clipfrac_avg": 0.07910472899675369, + "val/num_eos_tokens": 219, + "val/ratio": 0.9861233830451965, + "val/ratio_var": 8.373144373763353e-05 + }, + { + "episode": 12896, + "epoch": 0.6513131313131313, + "eps": 0, + "loss/policy_avg": -0.23323668539524078, + "loss/value_avg": 0.4607469439506531, + "lr": 2.9883044031688995e-06, + "objective/entropy": -85.76292419433594, + "objective/kl": 40.88092803955078, + "objective/non_score_reward": -2.044046401977539, + "objective/rlhf_reward": 14.174703598022461, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010081828804686666, + "policy/clipfrac_avg": 0.0041284579783678055, + "policy/entropy_avg": 0.1921253204345703, + "step": 806, + "val/clipfrac_avg": 0.16490784287452698, + "val/num_eos_tokens": 249, + "val/ratio": 0.9875838756561279, + "val/ratio_var": 6.450649379985407e-05 + }, + { + "episode": 12912, + "epoch": 0.6521212121212121, + "eps": 0, + "loss/policy_avg": -0.11845395714044571, + "loss/value_avg": 0.4592535197734833, + "lr": 2.976083284388031e-06, + "objective/entropy": -62.67930221557617, + "objective/kl": 33.47719955444336, + "objective/non_score_reward": -1.6738600730895996, + "objective/rlhf_reward": 12.771451950073242, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0011091704946011305, + "policy/clipfrac_avg": 0.00747725460678339, + "policy/entropy_avg": 0.1983814239501953, + "step": 807, + "val/clipfrac_avg": 0.033114466816186905, + "val/num_eos_tokens": 17, + "val/ratio": 0.9877710938453674, + "val/ratio_var": 6.532426050398499e-05 + }, + { + "episode": 12928, + "epoch": 0.6529292929292929, + "eps": 0, + "loss/policy_avg": 0.0469900444149971, + "loss/value_avg": 0.26954370737075806, + "lr": 2.9638766092746207e-06, + "objective/entropy": -88.9464111328125, + "objective/kl": 34.25027847290039, + "objective/non_score_reward": -1.7125139236450195, + "objective/rlhf_reward": 12.56873607635498, + "objective/scores": 14.25, + "policy/approxkl_avg": 0.0009448326891288161, + "policy/clipfrac_avg": 0.006387755274772644, + "policy/entropy_avg": 0.18566322326660156, + "step": 808, + "val/clipfrac_avg": 0.018177732825279236, + "val/num_eos_tokens": 761, + "val/ratio": 0.9882063269615173, + "val/ratio_var": 5.754105222877115e-05 + }, + { + "episode": 12944, + "epoch": 0.6537373737373737, + "eps": 0, + "loss/policy_avg": 0.00823296420276165, + "loss/value_avg": 0.44794195890426636, + "lr": 2.9516844649415165e-06, + "objective/entropy": -93.69944763183594, + "objective/kl": 40.86418151855469, + "objective/non_score_reward": -2.0432090759277344, + "objective/rlhf_reward": 13.542728424072266, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000799038156401366, + "policy/clipfrac_avg": 0.006408605724573135, + "policy/entropy_avg": 0.14969253540039062, + "step": 809, + "val/clipfrac_avg": 0.08976449072360992, + "val/num_eos_tokens": 215, + "val/ratio": 0.989780068397522, + "val/ratio_var": 6.0727201343979686e-05 + }, + { + "episode": 12960, + "epoch": 0.6545454545454545, + "eps": 0, + "loss/policy_avg": 0.024010583758354187, + "loss/value_avg": 0.3378852605819702, + "lr": 2.9395069383978725e-06, + "objective/entropy": -41.68758773803711, + "objective/kl": 35.73942184448242, + "objective/non_score_reward": -1.7869713306427002, + "objective/rlhf_reward": 15.314591407775879, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008846010314300656, + "policy/clipfrac_avg": 0.00692939106374979, + "policy/entropy_avg": 0.17317676544189453, + "step": 810, + "val/clipfrac_avg": 0.0032868448179215193, + "val/num_eos_tokens": 327, + "val/ratio": 0.9892865419387817, + "val/ratio_var": 4.3601932702586055e-05 + }, + { + "episode": 12976, + "epoch": 0.6553535353535354, + "eps": 0, + "loss/policy_avg": 0.1456170380115509, + "loss/value_avg": 0.5115708112716675, + "lr": 2.9273441165485227e-06, + "objective/entropy": -45.97090148925781, + "objective/kl": 36.01569366455078, + "objective/non_score_reward": -1.8007844686508179, + "objective/rlhf_reward": 14.23046588897705, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009762131376191974, + "policy/clipfrac_avg": 0.004803863354027271, + "policy/entropy_avg": 0.18830299377441406, + "step": 811, + "val/clipfrac_avg": 0.07713419198989868, + "val/num_eos_tokens": 267, + "val/ratio": 0.9878898859024048, + "val/ratio_var": 4.7962184908101335e-05 + }, + { + "episode": 12992, + "epoch": 0.6561616161616162, + "eps": 0, + "loss/policy_avg": 0.1002146378159523, + "loss/value_avg": 0.558029294013977, + "lr": 2.9151960861933616e-06, + "objective/entropy": -66.14154052734375, + "objective/kl": 36.481346130371094, + "objective/non_score_reward": -1.8240673542022705, + "objective/rlhf_reward": 14.175932884216309, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009823704604059458, + "policy/clipfrac_avg": 0.0034461808390915394, + "policy/entropy_avg": 0.1955432891845703, + "step": 812, + "val/clipfrac_avg": 0.10748951137065887, + "val/num_eos_tokens": 292, + "val/ratio": 0.986059308052063, + "val/ratio_var": 5.328569022822194e-05 + }, + { + "episode": 13008, + "epoch": 0.656969696969697, + "eps": 0, + "loss/policy_avg": -0.10509809851646423, + "loss/value_avg": 0.46006062626838684, + "lr": 2.9030629340267165e-06, + "objective/entropy": -110.846923828125, + "objective/kl": 36.021034240722656, + "objective/non_score_reward": -1.8010516166687012, + "objective/rlhf_reward": 16.33176040649414, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0008035163627937436, + "policy/clipfrac_avg": 0.005940265022218227, + "policy/entropy_avg": 0.15110206604003906, + "step": 813, + "val/clipfrac_avg": 0.07604548335075378, + "val/num_eos_tokens": 523, + "val/ratio": 0.9898133277893066, + "val/ratio_var": 6.0903592384420335e-05 + }, + { + "episode": 13024, + "epoch": 0.6577777777777778, + "eps": 0, + "loss/policy_avg": -0.11711955070495605, + "loss/value_avg": 1.2142528295516968, + "lr": 2.890944746636748e-06, + "objective/entropy": -40.75424575805664, + "objective/kl": 34.03726577758789, + "objective/non_score_reward": -1.701863408088684, + "objective/rlhf_reward": 13.091104507446289, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.000980376498773694, + "policy/clipfrac_avg": 0.008426186628639698, + "policy/entropy_avg": 0.20235157012939453, + "step": 814, + "val/clipfrac_avg": 0.02976190485060215, + "val/num_eos_tokens": 113, + "val/ratio": 0.9871314764022827, + "val/ratio_var": 6.27562912995927e-05 + }, + { + "episode": 13040, + "epoch": 0.6585858585858586, + "eps": 0, + "loss/policy_avg": -0.18143659830093384, + "loss/value_avg": 0.571576714515686, + "lr": 2.8788416105048124e-06, + "objective/entropy": -141.41366577148438, + "objective/kl": 42.53788757324219, + "objective/non_score_reward": -2.126894474029541, + "objective/rlhf_reward": 13.537168502807617, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008768361294642091, + "policy/clipfrac_avg": 0.006732264067977667, + "policy/entropy_avg": 0.17724037170410156, + "step": 815, + "val/clipfrac_avg": 0.03276197984814644, + "val/num_eos_tokens": 221, + "val/ratio": 0.9885480403900146, + "val/ratio_var": 8.229191007558256e-05 + }, + { + "episode": 13056, + "epoch": 0.6593939393939394, + "eps": 0, + "loss/policy_avg": 0.3017633855342865, + "loss/value_avg": 0.32543864846229553, + "lr": 2.8667536120048616e-06, + "objective/entropy": -61.10264587402344, + "objective/kl": 35.6143684387207, + "objective/non_score_reward": -1.7807183265686035, + "objective/rlhf_reward": 14.391157150268555, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0009194887243211269, + "policy/clipfrac_avg": 0.018624408170580864, + "policy/entropy_avg": 0.14903783798217773, + "step": 816, + "val/clipfrac_avg": 0.0073128631338477135, + "val/num_eos_tokens": 112, + "val/ratio": 0.989689290523529, + "val/ratio_var": 5.833535033161752e-05 + }, + { + "episode": 13072, + "epoch": 0.6602020202020202, + "eps": 0, + "loss/policy_avg": 0.06899265199899673, + "loss/value_avg": 0.6989455223083496, + "lr": 2.854680837402804e-06, + "objective/entropy": -117.86632537841797, + "objective/kl": 34.15174102783203, + "objective/non_score_reward": -1.7075870037078857, + "objective/rlhf_reward": 14.511162757873535, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009505614871159196, + "policy/clipfrac_avg": 0.008451011963188648, + "policy/entropy_avg": 0.1699972152709961, + "step": 817, + "val/clipfrac_avg": 0.000589622650295496, + "val/num_eos_tokens": 435, + "val/ratio": 0.9876314401626587, + "val/ratio_var": 9.73327478277497e-05 + }, + { + "episode": 13088, + "epoch": 0.661010101010101, + "eps": 0, + "loss/policy_avg": -0.05428570508956909, + "loss/value_avg": 0.2928343415260315, + "lr": 2.8426233728559172e-06, + "objective/entropy": -52.28473663330078, + "objective/kl": 40.512550354003906, + "objective/non_score_reward": -2.025627613067627, + "objective/rlhf_reward": 14.458747863769531, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0008845083648338914, + "policy/clipfrac_avg": 0.005094105843454599, + "policy/entropy_avg": 0.16685867309570312, + "step": 818, + "val/clipfrac_avg": 0.09271319210529327, + "val/num_eos_tokens": 32, + "val/ratio": 0.9885064959526062, + "val/ratio_var": 4.853576319874264e-05 + }, + { + "episode": 13104, + "epoch": 0.6618181818181819, + "eps": 0, + "loss/policy_avg": 0.12085426598787308, + "loss/value_avg": 0.5670195817947388, + "lr": 2.83058130441221e-06, + "objective/entropy": -196.3238983154297, + "objective/kl": 42.012672424316406, + "objective/non_score_reward": -2.1006336212158203, + "objective/rlhf_reward": 12.51655387878418, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0007904545636847615, + "policy/clipfrac_avg": 0.006419865880161524, + "policy/entropy_avg": 0.1437664031982422, + "step": 819, + "val/clipfrac_avg": 0.05388574302196503, + "val/num_eos_tokens": 496, + "val/ratio": 0.9899193644523621, + "val/ratio_var": 4.292546145734377e-05 + }, + { + "episode": 13120, + "epoch": 0.6626262626262627, + "eps": 0, + "loss/policy_avg": 0.6058844327926636, + "loss/value_avg": 0.7294787168502808, + "lr": 2.8185547180098195e-06, + "objective/entropy": -58.513553619384766, + "objective/kl": 38.73785400390625, + "objective/non_score_reward": -1.9368927478790283, + "objective/rlhf_reward": 12.883419036865234, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010288474150002003, + "policy/clipfrac_avg": 0.021498143672943115, + "policy/entropy_avg": 0.20896148681640625, + "step": 820, + "val/clipfrac_avg": 0.0723545029759407, + "val/num_eos_tokens": 134, + "val/ratio": 0.9887969493865967, + "val/ratio_var": 4.8135509132407606e-05 + }, + { + "episode": 13136, + "epoch": 0.6634343434343435, + "eps": 0, + "loss/policy_avg": -0.21932660043239594, + "loss/value_avg": 0.45023149251937866, + "lr": 2.8065436994763963e-06, + "objective/entropy": -38.43750762939453, + "objective/kl": 33.36050033569336, + "objective/non_score_reward": -1.6680251359939575, + "objective/rlhf_reward": 13.542912483215332, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0010698122205212712, + "policy/clipfrac_avg": 0.004931025672703981, + "policy/entropy_avg": 0.19058465957641602, + "step": 821, + "val/clipfrac_avg": 0.00852490495890379, + "val/num_eos_tokens": 190, + "val/ratio": 0.9864101409912109, + "val/ratio_var": 5.521172352018766e-05 + }, + { + "episode": 13152, + "epoch": 0.6642424242424242, + "eps": 0, + "loss/policy_avg": -0.01703064516186714, + "loss/value_avg": 0.2902008295059204, + "lr": 2.794548334528486e-06, + "objective/entropy": 13.10763931274414, + "objective/kl": 44.57916259765625, + "objective/non_score_reward": -2.2289581298828125, + "objective/rlhf_reward": 13.364792823791504, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0014038155786693096, + "policy/clipfrac_avg": 0.006509849801659584, + "policy/entropy_avg": 0.26366424560546875, + "step": 822, + "val/clipfrac_avg": 0.03128276392817497, + "val/num_eos_tokens": 35, + "val/ratio": 0.9824488162994385, + "val/ratio_var": 4.7705398174002767e-05 + }, + { + "episode": 13168, + "epoch": 0.665050505050505, + "eps": 0, + "loss/policy_avg": -0.02819274365901947, + "loss/value_avg": 0.2861873507499695, + "lr": 2.782568708770933e-06, + "objective/entropy": -94.84840393066406, + "objective/kl": 42.75294494628906, + "objective/non_score_reward": -2.1376476287841797, + "objective/rlhf_reward": 12.26860237121582, + "objective/scores": 14.375, + "policy/approxkl_avg": 0.0010491260327398777, + "policy/clipfrac_avg": 0.004985281266272068, + "policy/entropy_avg": 0.2000885009765625, + "step": 823, + "val/clipfrac_avg": 0.00754884909838438, + "val/num_eos_tokens": 289, + "val/ratio": 0.9866741895675659, + "val/ratio_var": 4.918476770399138e-05 + }, + { + "episode": 13184, + "epoch": 0.6658585858585858, + "eps": 0, + "loss/policy_avg": -0.024818487465381622, + "loss/value_avg": 0.3611629605293274, + "lr": 2.7706049076962483e-06, + "objective/entropy": -104.7535400390625, + "objective/kl": 45.08838653564453, + "objective/non_score_reward": -2.2544193267822266, + "objective/rlhf_reward": 12.214330673217773, + "objective/scores": 14.5, + "policy/approxkl_avg": 0.0009155337465927005, + "policy/clipfrac_avg": 0.008831838145852089, + "policy/entropy_avg": 0.16748523712158203, + "step": 824, + "val/clipfrac_avg": 0.0024258445482701063, + "val/num_eos_tokens": 163, + "val/ratio": 0.9890375137329102, + "val/ratio_var": 5.131318175699562e-05 + }, + { + "episode": 13200, + "epoch": 0.6666666666666666, + "eps": 0, + "loss/policy_avg": -0.28224268555641174, + "loss/value_avg": 0.8720104694366455, + "lr": 2.7586570166840154e-06, + "objective/entropy": -74.55836486816406, + "objective/kl": 30.59856605529785, + "objective/non_score_reward": -1.52992844581604, + "objective/rlhf_reward": 15.329446792602539, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008330290438607335, + "policy/clipfrac_avg": 0.010464916005730629, + "policy/entropy_avg": 0.15100550651550293, + "step": 825, + "val/clipfrac_avg": 0.08580771088600159, + "val/num_eos_tokens": 460, + "val/ratio": 0.9896235466003418, + "val/ratio_var": 9.328931628260761e-05 + }, + { + "episode": 13216, + "epoch": 0.6674747474747474, + "eps": 0, + "loss/policy_avg": -0.1715128868818283, + "loss/value_avg": 0.4540337920188904, + "lr": 2.746725121000273e-06, + "objective/entropy": -43.89616394042969, + "objective/kl": 41.760223388671875, + "objective/non_score_reward": -2.0880112648010254, + "objective/rlhf_reward": 13.872926712036133, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010317859705537558, + "policy/clipfrac_avg": 0.0054151020012795925, + "policy/entropy_avg": 0.19500446319580078, + "step": 826, + "val/clipfrac_avg": 0.03631855547428131, + "val/num_eos_tokens": 334, + "val/ratio": 0.9874616265296936, + "val/ratio_var": 6.214805034687743e-05 + }, + { + "episode": 13232, + "epoch": 0.6682828282828283, + "eps": 0, + "loss/policy_avg": -0.077702596783638, + "loss/value_avg": 0.6576616168022156, + "lr": 2.734809305796915e-06, + "objective/entropy": -13.071683883666992, + "objective/kl": 34.05264663696289, + "objective/non_score_reward": -1.7026324272155762, + "objective/rlhf_reward": 14.656742095947266, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0012533586705103517, + "policy/clipfrac_avg": 0.00517519935965538, + "policy/entropy_avg": 0.21735763549804688, + "step": 827, + "val/clipfrac_avg": 0.09026823937892914, + "val/num_eos_tokens": 468, + "val/ratio": 0.9863014221191406, + "val/ratio_var": 0.00010488701082067564 + }, + { + "episode": 13248, + "epoch": 0.6690909090909091, + "eps": 0, + "loss/policy_avg": -0.11604561656713486, + "loss/value_avg": 0.18491807579994202, + "lr": 2.7229096561110703e-06, + "objective/entropy": -132.30560302734375, + "objective/kl": 37.0118522644043, + "objective/non_score_reward": -1.8505926132202148, + "objective/rlhf_reward": 15.282218933105469, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008336640894412994, + "policy/clipfrac_avg": 0.0030343476682901382, + "policy/entropy_avg": 0.16221046447753906, + "step": 828, + "val/clipfrac_avg": 0.07020965218544006, + "val/num_eos_tokens": 41, + "val/ratio": 0.9894965291023254, + "val/ratio_var": 4.505051037995145e-05 + }, + { + "episode": 13264, + "epoch": 0.6698989898989899, + "eps": 0, + "loss/policy_avg": -0.23082350194454193, + "loss/value_avg": 0.9338909387588501, + "lr": 2.7110262568645063e-06, + "objective/entropy": -83.5711441040039, + "objective/kl": 42.80398941040039, + "objective/non_score_reward": -2.1401994228363037, + "objective/rlhf_reward": 13.555112838745117, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0008490884792990983, + "policy/clipfrac_avg": 0.024211544543504715, + "policy/entropy_avg": 0.16208171844482422, + "step": 829, + "val/clipfrac_avg": 0.01703277789056301, + "val/num_eos_tokens": 407, + "val/ratio": 0.9889751672744751, + "val/ratio_var": 6.549843965331092e-05 + }, + { + "episode": 13280, + "epoch": 0.6707070707070707, + "eps": 0, + "loss/policy_avg": -0.1790890097618103, + "loss/value_avg": 0.9402360320091248, + "lr": 2.699159192863018e-06, + "objective/entropy": -91.52725219726562, + "objective/kl": 30.28129005432129, + "objective/non_score_reward": -1.5140645503997803, + "objective/rlhf_reward": 13.71249771118164, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0006465939804911613, + "policy/clipfrac_avg": 0.003674568608403206, + "policy/entropy_avg": 0.14063799381256104, + "step": 830, + "val/clipfrac_avg": 0.05747377499938011, + "val/num_eos_tokens": 250, + "val/ratio": 0.9919025897979736, + "val/ratio_var": 6.961268809391186e-05 + }, + { + "episode": 13296, + "epoch": 0.6715151515151515, + "eps": 0, + "loss/policy_avg": 0.3400639295578003, + "loss/value_avg": 1.9688762426376343, + "lr": 2.687308548795825e-06, + "objective/entropy": -54.39606475830078, + "objective/kl": 44.12242889404297, + "objective/non_score_reward": -2.2061214447021484, + "objective/rlhf_reward": 14.004816055297852, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0009418983245268464, + "policy/clipfrac_avg": 0.009523827582597733, + "policy/entropy_avg": 0.18965661525726318, + "step": 831, + "val/clipfrac_avg": 0.045346200466156006, + "val/num_eos_tokens": 245, + "val/ratio": 0.9880996942520142, + "val/ratio_var": 6.565819057868794e-05 + }, + { + "episode": 13312, + "epoch": 0.6723232323232323, + "eps": 0, + "loss/policy_avg": 0.11609108746051788, + "loss/value_avg": 0.5635161399841309, + "lr": 2.675474409234965e-06, + "objective/entropy": -66.36386108398438, + "objective/kl": 38.93806838989258, + "objective/non_score_reward": -1.9469034671783447, + "objective/rlhf_reward": 14.076534271240234, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008638559957034886, + "policy/clipfrac_avg": 0.005322727840393782, + "policy/entropy_avg": 0.15152549743652344, + "step": 832, + "val/clipfrac_avg": 0.04381619766354561, + "val/num_eos_tokens": 20, + "val/ratio": 0.9889363050460815, + "val/ratio_var": 3.002676021424122e-05 + }, + { + "episode": 13328, + "epoch": 0.6731313131313131, + "eps": 0, + "loss/policy_avg": -0.03308400511741638, + "loss/value_avg": 0.3210276961326599, + "lr": 2.66365685863469e-06, + "objective/entropy": -111.39958953857422, + "objective/kl": 47.16798400878906, + "objective/non_score_reward": -2.3583993911743164, + "objective/rlhf_reward": 11.7197265625, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0009453936363570392, + "policy/clipfrac_avg": 0.004603059496730566, + "policy/entropy_avg": 0.18264532089233398, + "step": 833, + "val/clipfrac_avg": 0.062541663646698, + "val/num_eos_tokens": 763, + "val/ratio": 0.9879401922225952, + "val/ratio_var": 6.487595965154469e-05 + }, + { + "episode": 13344, + "epoch": 0.673939393939394, + "eps": 0, + "loss/policy_avg": 0.051825061440467834, + "loss/value_avg": 0.8246160745620728, + "lr": 2.651855981330872e-06, + "objective/entropy": -69.66492462158203, + "objective/kl": 37.68748474121094, + "objective/non_score_reward": -1.8843741416931152, + "objective/rlhf_reward": 12.764063835144043, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0010055399034172297, + "policy/clipfrac_avg": 0.004349449649453163, + "policy/entropy_avg": 0.18214893341064453, + "step": 834, + "val/clipfrac_avg": 0.06374436616897583, + "val/num_eos_tokens": 38, + "val/ratio": 0.9867125749588013, + "val/ratio_var": 9.083253826247528e-05 + }, + { + "episode": 13360, + "epoch": 0.6747474747474748, + "eps": 0, + "loss/policy_avg": -0.043558381497859955, + "loss/value_avg": 0.6093795895576477, + "lr": 2.6400718615403852e-06, + "objective/entropy": -108.79554748535156, + "objective/kl": 39.288482666015625, + "objective/non_score_reward": -1.9644241333007812, + "objective/rlhf_reward": 14.504325866699219, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0010659170802682638, + "policy/clipfrac_avg": 0.005707671865820885, + "policy/entropy_avg": 0.1986398696899414, + "step": 835, + "val/clipfrac_avg": 0.013371335342526436, + "val/num_eos_tokens": 647, + "val/ratio": 0.9869523048400879, + "val/ratio_var": 8.708865061635152e-05 + }, + { + "episode": 13376, + "epoch": 0.6755555555555556, + "eps": 0, + "loss/policy_avg": 0.09040576219558716, + "loss/value_avg": 0.4517725706100464, + "lr": 2.62830458336052e-06, + "objective/entropy": -59.1908073425293, + "objective/kl": 42.06044387817383, + "objective/non_score_reward": -2.1030220985412598, + "objective/rlhf_reward": 13.842290878295898, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.001016529044136405, + "policy/clipfrac_avg": 0.007354079745709896, + "policy/entropy_avg": 0.20728302001953125, + "step": 836, + "val/clipfrac_avg": 0.030187882483005524, + "val/num_eos_tokens": 477, + "val/ratio": 0.9865027666091919, + "val/ratio_var": 5.405073534348048e-05 + }, + { + "episode": 13392, + "epoch": 0.6763636363636364, + "eps": 0, + "loss/policy_avg": -0.21371892094612122, + "loss/value_avg": 0.26992350816726685, + "lr": 2.6165542307683744e-06, + "objective/entropy": -99.01640319824219, + "objective/kl": 43.636474609375, + "objective/non_score_reward": -2.18182373046875, + "objective/rlhf_reward": 13.08380126953125, + "objective/scores": 15.25, + "policy/approxkl_avg": 0.0009176525054499507, + "policy/clipfrac_avg": 0.006078221835196018, + "policy/entropy_avg": 0.1714763641357422, + "step": 837, + "val/clipfrac_avg": 0.025651777163147926, + "val/num_eos_tokens": 453, + "val/ratio": 0.988213300704956, + "val/ratio_var": 6.253684114199132e-05 + }, + { + "episode": 13408, + "epoch": 0.6771717171717172, + "eps": 0, + "loss/policy_avg": 0.08225658535957336, + "loss/value_avg": 0.4106031656265259, + "lr": 2.604820887620254e-06, + "objective/entropy": -62.21269989013672, + "objective/kl": 40.305843353271484, + "objective/non_score_reward": -2.015292167663574, + "objective/rlhf_reward": 15.75814437866211, + "objective/scores": 17.75, + "policy/approxkl_avg": 0.0009111049002967775, + "policy/clipfrac_avg": 0.005017759278416634, + "policy/entropy_avg": 0.18220901489257812, + "step": 838, + "val/clipfrac_avg": 0.0028512999415397644, + "val/num_eos_tokens": 268, + "val/ratio": 0.9878619909286499, + "val/ratio_var": 3.237010969314724e-05 + }, + { + "episode": 13424, + "epoch": 0.677979797979798, + "eps": 0, + "loss/policy_avg": 0.01355072483420372, + "loss/value_avg": 0.4876829981803894, + "lr": 2.5931046376510875e-06, + "objective/entropy": -38.304931640625, + "objective/kl": 35.24858093261719, + "objective/non_score_reward": -1.7624289989471436, + "objective/rlhf_reward": 12.284445762634277, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0008708706009201705, + "policy/clipfrac_avg": 0.005640837829560041, + "policy/entropy_avg": 0.16802191734313965, + "step": 839, + "val/clipfrac_avg": 0.029899097979068756, + "val/num_eos_tokens": 258, + "val/ratio": 0.989201009273529, + "val/ratio_var": 5.1689257816178724e-05 + }, + { + "episode": 13440, + "epoch": 0.6787878787878788, + "eps": 0, + "loss/policy_avg": 0.1623387336730957, + "loss/value_avg": 0.3584904074668884, + "lr": 2.5814055644738013e-06, + "objective/entropy": -63.923397064208984, + "objective/kl": 36.323509216308594, + "objective/non_score_reward": -1.8161752223968506, + "objective/rlhf_reward": 12.97288703918457, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0009589061955921352, + "policy/clipfrac_avg": 0.006033012643456459, + "policy/entropy_avg": 0.17229270935058594, + "step": 840, + "val/clipfrac_avg": 0.11569125950336456, + "val/num_eos_tokens": 159, + "val/ratio": 0.988390326499939, + "val/ratio_var": 7.233954966068268e-05 + }, + { + "episode": 13456, + "epoch": 0.6795959595959596, + "eps": 0, + "loss/policy_avg": -0.29571330547332764, + "loss/value_avg": 0.5451829433441162, + "lr": 2.5697237515787564e-06, + "objective/entropy": -159.0351104736328, + "objective/kl": 34.90948486328125, + "objective/non_score_reward": -1.7454744577407837, + "objective/rlhf_reward": 13.145150184631348, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0007017417810857296, + "policy/clipfrac_avg": 0.005396063905209303, + "policy/entropy_avg": 0.12675732374191284, + "step": 841, + "val/clipfrac_avg": 0.019056372344493866, + "val/num_eos_tokens": 508, + "val/ratio": 0.9914215207099915, + "val/ratio_var": 8.740482735447586e-05 + }, + { + "episode": 13472, + "epoch": 0.6804040404040405, + "eps": 0, + "loss/policy_avg": -0.33980700373649597, + "loss/value_avg": 0.5796597003936768, + "lr": 2.5580592823331237e-06, + "objective/entropy": -83.5593490600586, + "objective/kl": 36.281490325927734, + "objective/non_score_reward": -1.8140745162963867, + "objective/rlhf_reward": 14.18592643737793, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009474118705838919, + "policy/clipfrac_avg": 0.004876702558249235, + "policy/entropy_avg": 0.14034080505371094, + "step": 842, + "val/clipfrac_avg": 0.03151373565196991, + "val/num_eos_tokens": 208, + "val/ratio": 0.9901444911956787, + "val/ratio_var": 6.344315625028685e-05 + }, + { + "episode": 13488, + "epoch": 0.6812121212121212, + "eps": 0, + "loss/policy_avg": 0.11847849935293198, + "loss/value_avg": 0.3053319454193115, + "lr": 2.5464122399803126e-06, + "objective/entropy": -74.54005432128906, + "objective/kl": 46.23011779785156, + "objective/non_score_reward": -2.3115057945251465, + "objective/rlhf_reward": 12.993181228637695, + "objective/scores": 15.3125, + "policy/approxkl_avg": 0.0009884861065074801, + "policy/clipfrac_avg": 0.006667924579232931, + "policy/entropy_avg": 0.19658470153808594, + "step": 843, + "val/clipfrac_avg": 0.00010080645006382838, + "val/num_eos_tokens": 52, + "val/ratio": 0.9878820180892944, + "val/ratio_var": 4.166604412603192e-05 + }, + { + "episode": 13504, + "epoch": 0.682020202020202, + "eps": 0, + "loss/policy_avg": 0.07477365434169769, + "loss/value_avg": 0.717211902141571, + "lr": 2.5347827076393505e-06, + "objective/entropy": -55.136016845703125, + "objective/kl": 36.799835205078125, + "objective/non_score_reward": -1.8399916887283325, + "objective/rlhf_reward": 12.995945930480957, + "objective/scores": 14.8125, + "policy/approxkl_avg": 0.0010256943060085177, + "policy/clipfrac_avg": 0.009538116864860058, + "policy/entropy_avg": 0.19795989990234375, + "step": 844, + "val/clipfrac_avg": 0.0769520252943039, + "val/num_eos_tokens": 573, + "val/ratio": 0.9861817955970764, + "val/ratio_var": 7.031750283204019e-05 + }, + { + "episode": 13520, + "epoch": 0.6828282828282828, + "eps": 0, + "loss/policy_avg": -0.010380454361438751, + "loss/value_avg": 0.8978620767593384, + "lr": 2.5231707683043204e-06, + "objective/entropy": -164.994873046875, + "objective/kl": 41.09418487548828, + "objective/non_score_reward": -2.0547094345092773, + "objective/rlhf_reward": 12.367165565490723, + "objective/scores": 14.4375, + "policy/approxkl_avg": 0.0006862258305773139, + "policy/clipfrac_avg": 0.005633420776575804, + "policy/entropy_avg": 0.136183500289917, + "step": 845, + "val/clipfrac_avg": 0.10761284828186035, + "val/num_eos_tokens": 289, + "val/ratio": 0.9907996654510498, + "val/ratio_var": 4.169603198533878e-05 + }, + { + "episode": 13536, + "epoch": 0.6836363636363636, + "eps": 0, + "loss/policy_avg": -0.0011942088603973389, + "loss/value_avg": 0.7805696129798889, + "lr": 2.5115765048437445e-06, + "objective/entropy": -119.26876068115234, + "objective/kl": 37.73551940917969, + "objective/non_score_reward": -1.8867759704589844, + "objective/rlhf_reward": 13.769474029541016, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0007093651220202446, + "policy/clipfrac_avg": 0.028275154531002045, + "policy/entropy_avg": 0.1421506404876709, + "step": 846, + "val/clipfrac_avg": 0.08967090398073196, + "val/num_eos_tokens": 760, + "val/ratio": 0.9911968111991882, + "val/ratio_var": 3.498927253531292e-05 + }, + { + "episode": 13552, + "epoch": 0.6844444444444444, + "eps": 0, + "loss/policy_avg": -0.007019490003585815, + "loss/value_avg": 0.8198627233505249, + "lr": 2.5000000000000015e-06, + "objective/entropy": -89.728271484375, + "objective/kl": 34.76280975341797, + "objective/non_score_reward": -1.7381404638290405, + "objective/rlhf_reward": 13.925922393798828, + "objective/scores": 15.6875, + "policy/approxkl_avg": 0.0009024562314152718, + "policy/clipfrac_avg": 0.007501127198338509, + "policy/entropy_avg": 0.17042064666748047, + "step": 847, + "val/clipfrac_avg": 0.028372831642627716, + "val/num_eos_tokens": 17, + "val/ratio": 0.9899512529373169, + "val/ratio_var": 7.211479532998055e-05 + }, + { + "episode": 13568, + "epoch": 0.6852525252525252, + "eps": 0, + "loss/policy_avg": 0.015100933611392975, + "loss/value_avg": 0.25284868478775024, + "lr": 2.488441336388737e-06, + "objective/entropy": -184.20425415039062, + "objective/kl": 46.20475769042969, + "objective/non_score_reward": -2.3102383613586426, + "objective/rlhf_reward": 11.728824615478516, + "objective/scores": 14.0625, + "policy/approxkl_avg": 0.0007638686802238226, + "policy/clipfrac_avg": 0.005230149254202843, + "policy/entropy_avg": 0.15384864807128906, + "step": 848, + "val/clipfrac_avg": 0.01555095985531807, + "val/num_eos_tokens": 404, + "val/ratio": 0.9898365139961243, + "val/ratio_var": 5.412843165686354e-05 + }, + { + "episode": 13584, + "epoch": 0.686060606060606, + "eps": 0, + "loss/policy_avg": -0.13074953854084015, + "loss/value_avg": 0.5254482626914978, + "lr": 2.4769005964982718e-06, + "objective/entropy": -70.96700286865234, + "objective/kl": 37.31745147705078, + "objective/non_score_reward": -1.8658727407455444, + "objective/rlhf_reward": 13.665377616882324, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.000885509536601603, + "policy/clipfrac_avg": 0.006637534126639366, + "policy/entropy_avg": 0.17204856872558594, + "step": 849, + "val/clipfrac_avg": 0.050514861941337585, + "val/num_eos_tokens": 24, + "val/ratio": 0.9882212281227112, + "val/ratio_var": 5.447021248983219e-05 + }, + { + "episode": 13600, + "epoch": 0.6868686868686869, + "eps": 0, + "loss/policy_avg": 0.039310187101364136, + "loss/value_avg": 0.18585552275180817, + "lr": 2.465377862689019e-06, + "objective/entropy": -62.67339324951172, + "objective/kl": 48.39326477050781, + "objective/non_score_reward": -2.419663190841675, + "objective/rlhf_reward": 13.564711570739746, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0009431451326236129, + "policy/clipfrac_avg": 0.004864457994699478, + "policy/entropy_avg": 0.19204330444335938, + "step": 850, + "val/clipfrac_avg": 0.01945144683122635, + "val/num_eos_tokens": 36, + "val/ratio": 0.9872653484344482, + "val/ratio_var": 3.194732926203869e-05 + }, + { + "episode": 13616, + "epoch": 0.6876767676767677, + "eps": 0, + "loss/policy_avg": 0.07975710183382034, + "loss/value_avg": 0.478973388671875, + "lr": 2.4538732171928847e-06, + "objective/entropy": -76.03438568115234, + "objective/kl": 35.364593505859375, + "objective/non_score_reward": -1.7682294845581055, + "objective/rlhf_reward": 15.075519561767578, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008544789161533117, + "policy/clipfrac_avg": 0.005153194069862366, + "policy/entropy_avg": 0.16641998291015625, + "step": 851, + "val/clipfrac_avg": 0.03764945641160011, + "val/num_eos_tokens": 298, + "val/ratio": 0.9884979724884033, + "val/ratio_var": 4.7100515075726435e-05 + }, + { + "episode": 13632, + "epoch": 0.6884848484848485, + "eps": 0, + "loss/policy_avg": -0.10211583226919174, + "loss/value_avg": 0.7199238538742065, + "lr": 2.4423867421126923e-06, + "objective/entropy": -59.744590759277344, + "objective/kl": 35.15203857421875, + "objective/non_score_reward": -1.7576020956039429, + "objective/rlhf_reward": 15.05489730834961, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009704260155558586, + "policy/clipfrac_avg": 0.004688786808401346, + "policy/entropy_avg": 0.17905235290527344, + "step": 852, + "val/clipfrac_avg": 0.17230291664600372, + "val/num_eos_tokens": 644, + "val/ratio": 0.9872374534606934, + "val/ratio_var": 4.29930878453888e-05 + }, + { + "episode": 13648, + "epoch": 0.6892929292929293, + "eps": 0, + "loss/policy_avg": 0.0028443392366170883, + "loss/value_avg": 0.3421381115913391, + "lr": 2.4309185194215885e-06, + "objective/entropy": -63.859352111816406, + "objective/kl": 41.87160110473633, + "objective/non_score_reward": -2.0935800075531006, + "objective/rlhf_reward": 16.01579475402832, + "objective/scores": 18.125, + "policy/approxkl_avg": 0.0011341955978423357, + "policy/clipfrac_avg": 0.005558031145483255, + "policy/entropy_avg": 0.21439552307128906, + "step": 853, + "val/clipfrac_avg": 0.07718860357999802, + "val/num_eos_tokens": 400, + "val/ratio": 0.9858847856521606, + "val/ratio_var": 5.448433512356132e-05 + }, + { + "episode": 13664, + "epoch": 0.6901010101010101, + "eps": 0, + "loss/policy_avg": 0.11498575657606125, + "loss/value_avg": 0.909265398979187, + "lr": 2.4194686309624664e-06, + "objective/entropy": -138.73922729492188, + "objective/kl": 39.14732360839844, + "objective/non_score_reward": -1.9573662281036377, + "objective/rlhf_reward": 11.542634010314941, + "objective/scores": 13.5, + "policy/approxkl_avg": 0.0011413603788241744, + "policy/clipfrac_avg": 0.008165586739778519, + "policy/entropy_avg": 0.18193864822387695, + "step": 854, + "val/clipfrac_avg": 0.04511754959821701, + "val/num_eos_tokens": 264, + "val/ratio": 0.9880124926567078, + "val/ratio_var": 7.85917291068472e-05 + }, + { + "episode": 13680, + "epoch": 0.6909090909090909, + "eps": 0, + "loss/policy_avg": 0.4261026978492737, + "loss/value_avg": 1.0351085662841797, + "lr": 2.408037158447375e-06, + "objective/entropy": -80.8604736328125, + "objective/kl": 39.87467956542969, + "objective/non_score_reward": -1.9937340021133423, + "objective/rlhf_reward": 12.850015640258789, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.000854466634336859, + "policy/clipfrac_avg": 0.0031361770816147327, + "policy/entropy_avg": 0.1807117462158203, + "step": 855, + "val/clipfrac_avg": 0.06461638957262039, + "val/num_eos_tokens": 76, + "val/ratio": 0.9892903566360474, + "val/ratio_var": 6.158453470561653e-05 + }, + { + "episode": 13696, + "epoch": 0.6917171717171717, + "eps": 0, + "loss/policy_avg": 0.35515934228897095, + "loss/value_avg": 0.6004530787467957, + "lr": 2.3966241834569364e-06, + "objective/entropy": -109.00687408447266, + "objective/kl": 44.59175109863281, + "objective/non_score_reward": -2.2295875549316406, + "objective/rlhf_reward": 14.02041244506836, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0008301680209115148, + "policy/clipfrac_avg": 0.004295141436159611, + "policy/entropy_avg": 0.16374778747558594, + "step": 856, + "val/clipfrac_avg": 0.0578775517642498, + "val/num_eos_tokens": 355, + "val/ratio": 0.9892109632492065, + "val/ratio_var": 3.649172867881134e-05 + }, + { + "episode": 13712, + "epoch": 0.6925252525252525, + "eps": 0, + "loss/policy_avg": -0.07683120667934418, + "loss/value_avg": 0.7507062554359436, + "lr": 2.385229787439768e-06, + "objective/entropy": -65.75347137451172, + "objective/kl": 33.233463287353516, + "objective/non_score_reward": -1.6616731882095337, + "objective/rlhf_reward": 12.885202407836914, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0008445363491773605, + "policy/clipfrac_avg": 0.0028887931257486343, + "policy/entropy_avg": 0.1688997745513916, + "step": 857, + "val/clipfrac_avg": 0.029584139585494995, + "val/num_eos_tokens": 79, + "val/ratio": 0.9893004894256592, + "val/ratio_var": 4.4566000724444166e-05 + }, + { + "episode": 13728, + "epoch": 0.6933333333333334, + "eps": 0, + "loss/policy_avg": -0.04102202132344246, + "loss/value_avg": 0.295141339302063, + "lr": 2.3738540517118953e-06, + "objective/entropy": -137.21546936035156, + "objective/kl": 39.995941162109375, + "objective/non_score_reward": -1.9997971057891846, + "objective/rlhf_reward": 12.875202178955078, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0009317261283285916, + "policy/clipfrac_avg": 0.016368409618735313, + "policy/entropy_avg": 0.18145465850830078, + "step": 858, + "val/clipfrac_avg": 0.02603679709136486, + "val/num_eos_tokens": 133, + "val/ratio": 0.9879333972930908, + "val/ratio_var": 7.448039104929194e-05 + }, + { + "episode": 13744, + "epoch": 0.6941414141414142, + "eps": 0, + "loss/policy_avg": -0.09325989335775375, + "loss/value_avg": 0.4778023660182953, + "lr": 2.3624970574561773e-06, + "objective/entropy": -126.39140319824219, + "objective/kl": 41.33527755737305, + "objective/non_score_reward": -2.0667638778686523, + "objective/rlhf_reward": 15.823860168457031, + "objective/scores": 17.875, + "policy/approxkl_avg": 0.0009032993111759424, + "policy/clipfrac_avg": 0.008262147195637226, + "policy/entropy_avg": 0.1809234619140625, + "step": 859, + "val/clipfrac_avg": 0.006358179729431868, + "val/num_eos_tokens": 198, + "val/ratio": 0.9881711602210999, + "val/ratio_var": 8.430143498117104e-05 + }, + { + "episode": 13760, + "epoch": 0.694949494949495, + "eps": 0, + "loss/policy_avg": -0.1992878019809723, + "loss/value_avg": 0.4450816214084625, + "lr": 2.3511588857217228e-06, + "objective/entropy": -82.32437133789062, + "objective/kl": 36.61549377441406, + "objective/non_score_reward": -1.8307745456695557, + "objective/rlhf_reward": 14.497350692749023, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0009405683958902955, + "policy/clipfrac_avg": 0.0031182412058115005, + "policy/entropy_avg": 0.1852397918701172, + "step": 860, + "val/clipfrac_avg": 0.04030741751194, + "val/num_eos_tokens": 674, + "val/ratio": 0.9874944686889648, + "val/ratio_var": 5.9595575294224545e-05 + }, + { + "episode": 13776, + "epoch": 0.6957575757575758, + "eps": 0, + "loss/policy_avg": -0.0025226399302482605, + "loss/value_avg": 0.6712653636932373, + "lr": 2.339839617423318e-06, + "objective/entropy": -97.76516723632812, + "objective/kl": 32.36253356933594, + "objective/non_score_reward": -1.6181267499923706, + "objective/rlhf_reward": 14.108436584472656, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0008775125024840236, + "policy/clipfrac_avg": 0.006763250567018986, + "policy/entropy_avg": 0.15996170043945312, + "step": 861, + "val/clipfrac_avg": 0.01572282984852791, + "val/num_eos_tokens": 755, + "val/ratio": 0.9897935390472412, + "val/ratio_var": 4.971764428773895e-05 + }, + { + "episode": 13792, + "epoch": 0.6965656565656566, + "eps": 0, + "loss/policy_avg": 0.08292508125305176, + "loss/value_avg": 0.5537270307540894, + "lr": 2.32853933334084e-06, + "objective/entropy": -61.261680603027344, + "objective/kl": 36.61561584472656, + "objective/non_score_reward": -1.8307808637619019, + "objective/rlhf_reward": 12.762969970703125, + "objective/scores": 14.625, + "policy/approxkl_avg": 0.0009761417168192565, + "policy/clipfrac_avg": 0.0026826506946235895, + "policy/entropy_avg": 0.22123146057128906, + "step": 862, + "val/clipfrac_avg": 0.044523607939481735, + "val/num_eos_tokens": 56, + "val/ratio": 0.9871484041213989, + "val/ratio_var": 7.335093687288463e-05 + }, + { + "episode": 13808, + "epoch": 0.6973737373737374, + "eps": 0, + "loss/policy_avg": 0.36910420656204224, + "loss/value_avg": 0.9228214025497437, + "lr": 2.317258114118686e-06, + "objective/entropy": -76.75859069824219, + "objective/kl": 36.92033004760742, + "objective/non_score_reward": -1.8460166454315186, + "objective/rlhf_reward": 14.872733116149902, + "objective/scores": 16.75, + "policy/approxkl_avg": 0.0009935328271239996, + "policy/clipfrac_avg": 0.006773949135094881, + "policy/entropy_avg": 0.1806344985961914, + "step": 863, + "val/clipfrac_avg": 0.07717607915401459, + "val/num_eos_tokens": 514, + "val/ratio": 0.9876723289489746, + "val/ratio_var": 6.279607623582706e-05 + }, + { + "episode": 13824, + "epoch": 0.6981818181818182, + "eps": 0, + "loss/policy_avg": -0.21535149216651917, + "loss/value_avg": 0.42513978481292725, + "lr": 2.305996040265198e-06, + "objective/entropy": -77.51583862304688, + "objective/kl": 45.520545959472656, + "objective/non_score_reward": -2.276027202606201, + "objective/rlhf_reward": 14.59897232055664, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0008458065567538142, + "policy/clipfrac_avg": 0.005439429543912411, + "policy/entropy_avg": 0.16998863220214844, + "step": 864, + "val/clipfrac_avg": 0.03808614984154701, + "val/num_eos_tokens": 93, + "val/ratio": 0.9894691109657288, + "val/ratio_var": 6.017706618877128e-05 + }, + { + "episode": 13840, + "epoch": 0.6989898989898989, + "eps": 0, + "loss/policy_avg": -0.08341407775878906, + "loss/value_avg": 0.7828294038772583, + "lr": 2.29475319215209e-06, + "objective/entropy": -82.54826354980469, + "objective/kl": 39.21794891357422, + "objective/non_score_reward": -1.9608973264694214, + "objective/rlhf_reward": 14.359415054321289, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007373163243755698, + "policy/clipfrac_avg": 0.003970882389694452, + "policy/entropy_avg": 0.14882469177246094, + "step": 865, + "val/clipfrac_avg": 0.17675819993019104, + "val/num_eos_tokens": 134, + "val/ratio": 0.9905297756195068, + "val/ratio_var": 5.841355596203357e-05 + }, + { + "episode": 13856, + "epoch": 0.6997979797979798, + "eps": 0, + "loss/policy_avg": 0.029253050684928894, + "loss/value_avg": 0.8540998101234436, + "lr": 2.2835296500138664e-06, + "objective/entropy": -121.09165954589844, + "objective/kl": 34.1153564453125, + "objective/non_score_reward": -1.7057678699493408, + "objective/rlhf_reward": 13.333294868469238, + "objective/scores": 15.0625, + "policy/approxkl_avg": 0.0009264532127417624, + "policy/clipfrac_avg": 0.0050398847088217735, + "policy/entropy_avg": 0.18099594116210938, + "step": 866, + "val/clipfrac_avg": 0.02329857461154461, + "val/num_eos_tokens": 1353, + "val/ratio": 0.9883856177330017, + "val/ratio_var": 8.088548202067614e-05 + }, + { + "episode": 13872, + "epoch": 0.7006060606060606, + "eps": 0, + "loss/policy_avg": -0.1498204916715622, + "loss/value_avg": 0.4047302007675171, + "lr": 2.272325493947257e-06, + "objective/entropy": -49.65830993652344, + "objective/kl": 36.36531066894531, + "objective/non_score_reward": -1.8182655572891235, + "objective/rlhf_reward": 14.494234085083008, + "objective/scores": 16.25, + "policy/approxkl_avg": 0.0010429773246869445, + "policy/clipfrac_avg": 0.005474097095429897, + "policy/entropy_avg": 0.18477344512939453, + "step": 867, + "val/clipfrac_avg": 0.0032526040449738503, + "val/num_eos_tokens": 101, + "val/ratio": 0.9877700805664062, + "val/ratio_var": 9.155944280792028e-05 + }, + { + "episode": 13888, + "epoch": 0.7014141414141414, + "eps": 0, + "loss/policy_avg": -0.1165279895067215, + "loss/value_avg": 0.7669879198074341, + "lr": 2.261140803910644e-06, + "objective/entropy": -77.34773254394531, + "objective/kl": 38.648643493652344, + "objective/non_score_reward": -1.9324320554733276, + "objective/rlhf_reward": 13.637880325317383, + "objective/scores": 15.5625, + "policy/approxkl_avg": 0.000937658129259944, + "policy/clipfrac_avg": 0.006235727574676275, + "policy/entropy_avg": 0.19645977020263672, + "step": 868, + "val/clipfrac_avg": 0.058441728353500366, + "val/num_eos_tokens": 310, + "val/ratio": 0.9879204034805298, + "val/ratio_var": 6.414015660993755e-05 + }, + { + "episode": 13904, + "epoch": 0.7022222222222222, + "eps": 0, + "loss/policy_avg": -0.016327757388353348, + "loss/value_avg": 0.43844515085220337, + "lr": 2.2499756597234872e-06, + "objective/entropy": -163.28765869140625, + "objective/kl": 48.81157684326172, + "objective/non_score_reward": -2.4405789375305176, + "objective/rlhf_reward": 11.70395278930664, + "objective/scores": 14.125, + "policy/approxkl_avg": 0.000844982685521245, + "policy/clipfrac_avg": 0.0032518147490918636, + "policy/entropy_avg": 0.16161727905273438, + "step": 869, + "val/clipfrac_avg": 0.014801801182329655, + "val/num_eos_tokens": 1949, + "val/ratio": 0.9885438680648804, + "val/ratio_var": 5.214377597440034e-05 + }, + { + "episode": 13920, + "epoch": 0.703030303030303, + "eps": 0, + "loss/policy_avg": -0.18964827060699463, + "loss/value_avg": 0.5136280655860901, + "lr": 2.238830141065765e-06, + "objective/entropy": -90.82677459716797, + "objective/kl": 40.3735237121582, + "objective/non_score_reward": -2.0186760425567627, + "objective/rlhf_reward": 15.153199195861816, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0008435281342826784, + "policy/clipfrac_avg": 0.005789198447018862, + "policy/entropy_avg": 0.1680126190185547, + "step": 870, + "val/clipfrac_avg": 0.0648677870631218, + "val/num_eos_tokens": 555, + "val/ratio": 0.9890276193618774, + "val/ratio_var": 7.125848060240969e-05 + }, + { + "episode": 13936, + "epoch": 0.7038383838383838, + "eps": 0, + "loss/policy_avg": 0.02698252536356449, + "loss/value_avg": 0.2791609764099121, + "lr": 2.2277043274773856e-06, + "objective/entropy": -45.07421112060547, + "objective/kl": 39.21965026855469, + "objective/non_score_reward": -1.9609825611114502, + "objective/rlhf_reward": 14.624954223632812, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0009133699350059032, + "policy/clipfrac_avg": 0.004030972719192505, + "policy/entropy_avg": 0.18420028686523438, + "step": 871, + "val/clipfrac_avg": 0.03648557886481285, + "val/num_eos_tokens": 232, + "val/ratio": 0.9881106615066528, + "val/ratio_var": 5.0894170271931216e-05 + }, + { + "episode": 13952, + "epoch": 0.7046464646464646, + "eps": 0, + "loss/policy_avg": -0.12599462270736694, + "loss/value_avg": 0.5007761716842651, + "lr": 2.216598298357646e-06, + "objective/entropy": -52.43424987792969, + "objective/kl": 37.084686279296875, + "objective/non_score_reward": -1.8542344570159912, + "objective/rlhf_reward": 14.18482780456543, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008421384263783693, + "policy/clipfrac_avg": 0.002664695493876934, + "policy/entropy_avg": 0.1716923713684082, + "step": 872, + "val/clipfrac_avg": 0.0010017228778451681, + "val/num_eos_tokens": 274, + "val/ratio": 0.9881088137626648, + "val/ratio_var": 6.610793934669346e-05 + }, + { + "episode": 13968, + "epoch": 0.7054545454545454, + "eps": 0, + "loss/policy_avg": 0.35178622603416443, + "loss/value_avg": 0.5846590995788574, + "lr": 2.2055121329646416e-06, + "objective/entropy": -138.92434692382812, + "objective/kl": 41.75783157348633, + "objective/non_score_reward": -2.0878915786743164, + "objective/rlhf_reward": 13.091796875, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0008433774346485734, + "policy/clipfrac_avg": 0.006052501033991575, + "policy/entropy_avg": 0.16450119018554688, + "step": 873, + "val/clipfrac_avg": 0.0322103314101696, + "val/num_eos_tokens": 1144, + "val/ratio": 0.9888837337493896, + "val/ratio_var": 5.522933497559279e-05 + }, + { + "episode": 13984, + "epoch": 0.7062626262626263, + "eps": 0, + "loss/policy_avg": -0.004525661468505859, + "loss/value_avg": 0.5259770750999451, + "lr": 2.1944459104147188e-06, + "objective/entropy": -122.14696502685547, + "objective/kl": 37.81830978393555, + "objective/non_score_reward": -1.8909155130386353, + "objective/rlhf_reward": 13.296585083007812, + "objective/scores": 15.1875, + "policy/approxkl_avg": 0.0006640740903094411, + "policy/clipfrac_avg": 0.004708480089902878, + "policy/entropy_avg": 0.14041614532470703, + "step": 874, + "val/clipfrac_avg": 0.10728074610233307, + "val/num_eos_tokens": 35, + "val/ratio": 0.990477979183197, + "val/ratio_var": 6.866270996397361e-05 + }, + { + "episode": 14000, + "epoch": 0.7070707070707071, + "eps": 0, + "loss/policy_avg": 0.11862924695014954, + "loss/value_avg": 1.0722770690917969, + "lr": 2.1833997096818897e-06, + "objective/entropy": -54.207332611083984, + "objective/kl": 37.62938690185547, + "objective/non_score_reward": -1.8814692497253418, + "objective/rlhf_reward": 14.2200927734375, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.000859625346492976, + "policy/clipfrac_avg": 0.005390159785747528, + "policy/entropy_avg": 0.1755199432373047, + "step": 875, + "val/clipfrac_avg": 0.0593750923871994, + "val/num_eos_tokens": 201, + "val/ratio": 0.9878208637237549, + "val/ratio_var": 6.484645564341918e-05 + }, + { + "episode": 14016, + "epoch": 0.7078787878787879, + "eps": 0, + "loss/policy_avg": -0.2030085027217865, + "loss/value_avg": 0.5594953298568726, + "lr": 2.1723736095972946e-06, + "objective/entropy": -16.795242309570312, + "objective/kl": 44.697776794433594, + "objective/non_score_reward": -2.234889030456543, + "objective/rlhf_reward": 13.140110969543457, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0011914006900042295, + "policy/clipfrac_avg": 0.005575358867645264, + "policy/entropy_avg": 0.2543792724609375, + "step": 876, + "val/clipfrac_avg": 0.02393021248281002, + "val/num_eos_tokens": 24, + "val/ratio": 0.9835648536682129, + "val/ratio_var": 0.00010361165914218873 + }, + { + "episode": 14032, + "epoch": 0.7086868686868687, + "eps": 0, + "loss/policy_avg": 0.05069049820303917, + "loss/value_avg": 0.6454532146453857, + "lr": 2.161367688848618e-06, + "objective/entropy": -105.85357666015625, + "objective/kl": 50.78942108154297, + "objective/non_score_reward": -2.53947114944458, + "objective/rlhf_reward": 13.077716827392578, + "objective/scores": 15.625, + "policy/approxkl_avg": 0.0008386933477595448, + "policy/clipfrac_avg": 0.003330180887132883, + "policy/entropy_avg": 0.1841726303100586, + "step": 877, + "val/clipfrac_avg": 0.010819870047271252, + "val/num_eos_tokens": 513, + "val/ratio": 0.9886084198951721, + "val/ratio_var": 5.4145024478202686e-05 + }, + { + "episode": 14048, + "epoch": 0.7094949494949495, + "eps": 0, + "loss/policy_avg": -0.32361558079719543, + "loss/value_avg": 0.4275820851325989, + "lr": 2.1503820259795357e-06, + "objective/entropy": -193.71432495117188, + "objective/kl": 41.63684844970703, + "objective/non_score_reward": -2.0818424224853516, + "objective/rlhf_reward": 14.261907577514648, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007008577231317759, + "policy/clipfrac_avg": 0.004103646147996187, + "policy/entropy_avg": 0.1445927619934082, + "step": 878, + "val/clipfrac_avg": 7.62939453125e-06, + "val/num_eos_tokens": 323, + "val/ratio": 0.9903497695922852, + "val/ratio_var": 3.22636442433577e-05 + }, + { + "episode": 14064, + "epoch": 0.7103030303030303, + "eps": 0, + "loss/policy_avg": -0.1645442247390747, + "loss/value_avg": 0.7777354717254639, + "lr": 2.139416699389153e-06, + "objective/entropy": -45.89549255371094, + "objective/kl": 34.282958984375, + "objective/non_score_reward": -1.7141478061676025, + "objective/rlhf_reward": 14.692102432250977, + "objective/scores": 16.375, + "policy/approxkl_avg": 0.0007650890620425344, + "policy/clipfrac_avg": 0.00561137031763792, + "policy/entropy_avg": 0.15566253662109375, + "step": 879, + "val/clipfrac_avg": 0.10208053886890411, + "val/num_eos_tokens": 49, + "val/ratio": 0.9895084500312805, + "val/ratio_var": 3.380152702447958e-05 + }, + { + "episode": 14080, + "epoch": 0.7111111111111111, + "eps": 0, + "loss/policy_avg": -0.023053357377648354, + "loss/value_avg": 0.521371603012085, + "lr": 2.128471787331446e-06, + "objective/entropy": -50.96327209472656, + "objective/kl": 41.30862808227539, + "objective/non_score_reward": -2.0654313564300537, + "objective/rlhf_reward": 14.450193405151367, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.0009377336828038096, + "policy/clipfrac_avg": 0.00421791011467576, + "policy/entropy_avg": 0.17507076263427734, + "step": 880, + "val/clipfrac_avg": 0.04448426887392998, + "val/num_eos_tokens": 368, + "val/ratio": 0.9889349937438965, + "val/ratio_var": 3.734078927664086e-05 + }, + { + "episode": 14096, + "epoch": 0.711919191919192, + "eps": 0, + "loss/policy_avg": -0.07007278501987457, + "loss/value_avg": 0.2162141352891922, + "lr": 2.117547367914705e-06, + "objective/entropy": -69.7264175415039, + "objective/kl": 41.05116653442383, + "objective/non_score_reward": -2.052558422088623, + "objective/rlhf_reward": 14.830253601074219, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.001040782080963254, + "policy/clipfrac_avg": 0.0033751465380191803, + "policy/entropy_avg": 0.20317459106445312, + "step": 881, + "val/clipfrac_avg": 0.03193364292383194, + "val/num_eos_tokens": 551, + "val/ratio": 0.9861003160476685, + "val/ratio_var": 8.361194340977818e-05 + }, + { + "episode": 14112, + "epoch": 0.7127272727272728, + "eps": 0, + "loss/policy_avg": -0.07847104221582413, + "loss/value_avg": 0.2657424807548523, + "lr": 2.1066435191009717e-06, + "objective/entropy": -86.86426544189453, + "objective/kl": 45.00962829589844, + "objective/non_score_reward": -2.250481367111206, + "objective/rlhf_reward": 13.913580894470215, + "objective/scores": 16.125, + "policy/approxkl_avg": 0.0010606081923469901, + "policy/clipfrac_avg": 0.0069802612997591496, + "policy/entropy_avg": 0.20689773559570312, + "step": 882, + "val/clipfrac_avg": 0.038475796580314636, + "val/num_eos_tokens": 546, + "val/ratio": 0.985522449016571, + "val/ratio_var": 8.580441499361768e-05 + }, + { + "episode": 14128, + "epoch": 0.7135353535353536, + "eps": 0, + "loss/policy_avg": -0.1323227733373642, + "loss/value_avg": 0.21754738688468933, + "lr": 2.095760318705487e-06, + "objective/entropy": -109.19290161132812, + "objective/kl": 42.61073303222656, + "objective/non_score_reward": -2.1305365562438965, + "objective/rlhf_reward": 14.447587966918945, + "objective/scores": 16.625, + "policy/approxkl_avg": 0.0010359378065913916, + "policy/clipfrac_avg": 0.006892787292599678, + "policy/entropy_avg": 0.20017433166503906, + "step": 883, + "val/clipfrac_avg": 0.010656789876520634, + "val/num_eos_tokens": 36, + "val/ratio": 0.9864689111709595, + "val/ratio_var": 4.844724026042968e-05 + }, + { + "episode": 14144, + "epoch": 0.7143434343434344, + "eps": 0, + "loss/policy_avg": -0.1507091522216797, + "loss/value_avg": 0.2665415108203888, + "lr": 2.0848978443961314e-06, + "objective/entropy": -119.52471923828125, + "objective/kl": 43.517696380615234, + "objective/non_score_reward": -2.17588472366333, + "objective/rlhf_reward": 14.300678253173828, + "objective/scores": 16.5, + "policy/approxkl_avg": 0.000938809709623456, + "policy/clipfrac_avg": 0.012131594121456146, + "policy/entropy_avg": 0.18040943145751953, + "step": 884, + "val/clipfrac_avg": 0.006075765937566757, + "val/num_eos_tokens": 620, + "val/ratio": 0.9873965978622437, + "val/ratio_var": 6.621635111514479e-05 + }, + { + "episode": 14160, + "epoch": 0.7151515151515152, + "eps": 0, + "loss/policy_avg": -0.028530046343803406, + "loss/value_avg": 0.434810072183609, + "lr": 2.074056173692881e-06, + "objective/entropy": -62.273380279541016, + "objective/kl": 45.77116394042969, + "objective/non_score_reward": -2.2885584831237793, + "objective/rlhf_reward": 14.617691040039062, + "objective/scores": 16.875, + "policy/approxkl_avg": 0.0009718666551634669, + "policy/clipfrac_avg": 0.004924894310534, + "policy/entropy_avg": 0.20779800415039062, + "step": 885, + "val/clipfrac_avg": 0.036309417337179184, + "val/num_eos_tokens": 179, + "val/ratio": 0.9870821237564087, + "val/ratio_var": 8.347785478690639e-05 + }, + { + "episode": 14176, + "epoch": 0.7159595959595959, + "eps": 0, + "loss/policy_avg": -0.15033547580242157, + "loss/value_avg": 0.6043248772621155, + "lr": 2.0632353839672415e-06, + "objective/entropy": -84.31056213378906, + "objective/kl": 32.41828918457031, + "objective/non_score_reward": -1.6209145784378052, + "objective/rlhf_reward": 15.332210540771484, + "objective/scores": 17.0, + "policy/approxkl_avg": 0.0008589382632635534, + "policy/clipfrac_avg": 0.002752305706962943, + "policy/entropy_avg": 0.18497180938720703, + "step": 886, + "val/clipfrac_avg": 0.03947104513645172, + "val/num_eos_tokens": 180, + "val/ratio": 0.9879313707351685, + "val/ratio_var": 5.601351585937664e-05 + }, + { + "episode": 14192, + "epoch": 0.7167676767676767, + "eps": 0, + "loss/policy_avg": 0.2279771864414215, + "loss/value_avg": 1.2074341773986816, + "lr": 2.0524355524417017e-06, + "objective/entropy": -51.082984924316406, + "objective/kl": 36.84368133544922, + "objective/non_score_reward": -1.842184066772461, + "objective/rlhf_reward": 14.087503433227539, + "objective/scores": 15.9375, + "policy/approxkl_avg": 0.0010461971396580338, + "policy/clipfrac_avg": 0.004587469156831503, + "policy/entropy_avg": 0.22597980499267578, + "step": 887, + "val/clipfrac_avg": 0.015519710257649422, + "val/num_eos_tokens": 448, + "val/ratio": 0.9849185943603516, + "val/ratio_var": 6.34926327620633e-05 + }, + { + "episode": 14208, + "epoch": 0.7175757575757575, + "eps": 0, + "loss/policy_avg": -0.026267291978001595, + "loss/value_avg": 0.5965927839279175, + "lr": 2.041656756189184e-06, + "objective/entropy": -154.47012329101562, + "objective/kl": 42.720699310302734, + "objective/non_score_reward": -2.1360349655151367, + "objective/rlhf_reward": 14.957715034484863, + "objective/scores": 17.125, + "policy/approxkl_avg": 0.0009299394441768527, + "policy/clipfrac_avg": 0.008321288973093033, + "policy/entropy_avg": 0.1751893162727356, + "step": 888, + "val/clipfrac_avg": 0.01875000074505806, + "val/num_eos_tokens": 428, + "val/ratio": 0.9883604049682617, + "val/ratio_var": 7.325677870539948e-05 + }, + { + "episode": 14224, + "epoch": 0.7183838383838383, + "eps": 0, + "loss/policy_avg": 0.40078702569007874, + "loss/value_avg": 1.1534854173660278, + "lr": 2.030899072132493e-06, + "objective/entropy": -128.48831176757812, + "objective/kl": 33.898155212402344, + "objective/non_score_reward": -1.6949079036712646, + "objective/rlhf_reward": 12.598060607910156, + "objective/scores": 14.3125, + "policy/approxkl_avg": 0.0006705562118440866, + "policy/clipfrac_avg": 0.005831941030919552, + "policy/entropy_avg": 0.21662330627441406, + "step": 889, + "val/clipfrac_avg": 0.02483724057674408, + "val/num_eos_tokens": 47, + "val/ratio": 0.9912885427474976, + "val/ratio_var": 7.058407936710864e-05 + }, + { + "episode": 14240, + "epoch": 0.7191919191919192, + "eps": 0, + "loss/policy_avg": -0.0855930745601654, + "loss/value_avg": 0.32921552658081055, + "lr": 2.0201625770437647e-06, + "objective/entropy": -86.9806137084961, + "objective/kl": 40.06652069091797, + "objective/non_score_reward": -2.003325939178467, + "objective/rlhf_reward": 14.035737037658691, + "objective/scores": 16.0, + "policy/approxkl_avg": 0.0008815613109618425, + "policy/clipfrac_avg": 0.006017718929797411, + "policy/entropy_avg": 0.1755352020263672, + "step": 890, + "val/clipfrac_avg": 0.06256753951311111, + "val/num_eos_tokens": 20, + "val/ratio": 0.9872984290122986, + "val/ratio_var": 8.527067984687164e-05 + }, + { + "episode": 14256, + "epoch": 0.72, + "eps": 0, + "loss/policy_avg": -0.17781905829906464, + "loss/value_avg": 0.2275725156068802, + "lr": 2.00944734754392e-06, + "objective/entropy": -99.16207122802734, + "objective/kl": 41.95650863647461, + "objective/non_score_reward": -2.097825288772583, + "objective/rlhf_reward": 13.74592399597168, + "objective/scores": 15.875, + "policy/approxkl_avg": 0.0007897773757576942, + "policy/clipfrac_avg": 0.0043031517416238785, + "policy/entropy_avg": 0.1585559844970703, + "step": 891, + "val/clipfrac_avg": 0.012334126979112625, + "val/num_eos_tokens": 424, + "val/ratio": 0.9894897937774658, + "val/ratio_var": 5.430886085378006e-05 + }, + { + "episode": 14272, + "epoch": 0.7208080808080808, + "eps": 0, + "loss/policy_avg": -0.36396488547325134, + "loss/value_avg": 0.8662234544754028, + "lr": 1.9987534601021225e-06, + "objective/entropy": -61.42802047729492, + "objective/kl": 37.07469177246094, + "objective/non_score_reward": -1.8537347316741943, + "objective/rlhf_reward": 13.044702529907227, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0008486385922878981, + "policy/clipfrac_avg": 0.006757801398634911, + "policy/entropy_avg": 0.1866321563720703, + "step": 892, + "val/clipfrac_avg": 0.054801926016807556, + "val/num_eos_tokens": 584, + "val/ratio": 0.9883260726928711, + "val/ratio_var": 6.7303488322068e-05 + }, + { + "episode": 14288, + "epoch": 0.7216161616161616, + "eps": 0, + "loss/policy_avg": -0.12171509861946106, + "loss/value_avg": 0.3499939441680908, + "lr": 1.988080991035223e-06, + "objective/entropy": -58.38874053955078, + "objective/kl": 42.03372573852539, + "objective/non_score_reward": -2.1016862392425537, + "objective/rlhf_reward": 12.828001022338867, + "objective/scores": 14.9375, + "policy/approxkl_avg": 0.0008078413084149361, + "policy/clipfrac_avg": 0.004710954613983631, + "policy/entropy_avg": 0.16219139099121094, + "step": 893, + "val/clipfrac_avg": 0.03385913744568825, + "val/num_eos_tokens": 211, + "val/ratio": 0.9909918904304504, + "val/ratio_var": 4.974031253368594e-05 + }, + { + "episode": 14304, + "epoch": 0.7224242424242424, + "eps": 0, + "loss/policy_avg": 0.15615974366664886, + "loss/value_avg": 0.4985978901386261, + "lr": 1.977430016507222e-06, + "objective/entropy": -72.03929138183594, + "objective/kl": 43.86260986328125, + "objective/non_score_reward": -2.1931304931640625, + "objective/rlhf_reward": 13.322494506835938, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0009859772399067879, + "policy/clipfrac_avg": 0.004861406981945038, + "policy/entropy_avg": 0.19806480407714844, + "step": 894, + "val/clipfrac_avg": 0.008869262412190437, + "val/num_eos_tokens": 389, + "val/ratio": 0.9869673252105713, + "val/ratio_var": 5.9525649703573436e-05 + }, + { + "episode": 14320, + "epoch": 0.7232323232323232, + "eps": 0, + "loss/policy_avg": -0.018653089180588722, + "loss/value_avg": 0.3236011266708374, + "lr": 1.966800612528723e-06, + "objective/entropy": -30.8804874420166, + "objective/kl": 38.795677185058594, + "objective/non_score_reward": -1.9397838115692139, + "objective/rlhf_reward": 12.958653450012207, + "objective/scores": 14.875, + "policy/approxkl_avg": 0.0011067797895520926, + "policy/clipfrac_avg": 0.004753993824124336, + "policy/entropy_avg": 0.21401214599609375, + "step": 895, + "val/clipfrac_avg": 0.013285137712955475, + "val/num_eos_tokens": 190, + "val/ratio": 0.9844212532043457, + "val/ratio_var": 5.578770287684165e-05 + }, + { + "episode": 14336, + "epoch": 0.724040404040404, + "eps": 0, + "loss/policy_avg": -0.04005177319049835, + "loss/value_avg": 0.564488410949707, + "lr": 1.956192854956397e-06, + "objective/entropy": -88.01570129394531, + "objective/kl": 40.9593505859375, + "objective/non_score_reward": -2.0479674339294434, + "objective/rlhf_reward": 13.350469589233398, + "objective/scores": 15.375, + "policy/approxkl_avg": 0.0010309319477528334, + "policy/clipfrac_avg": 0.005483411252498627, + "policy/entropy_avg": 0.21037864685058594, + "step": 896, + "val/clipfrac_avg": 0.01046415138989687, + "val/num_eos_tokens": 539, + "val/ratio": 0.9865807890892029, + "val/ratio_var": 9.776368096936494e-05 + }, + { + "episode": 14352, + "epoch": 0.7248484848484849, + "eps": 0, + "loss/policy_avg": -0.1358407437801361, + "loss/value_avg": 0.536889910697937, + "lr": 1.945606819492429e-06, + "objective/entropy": -98.2094955444336, + "objective/kl": 37.47423553466797, + "objective/non_score_reward": -1.8737118244171143, + "objective/rlhf_reward": 13.634100914001465, + "objective/scores": 15.5, + "policy/approxkl_avg": 0.0007985340780578554, + "policy/clipfrac_avg": 0.005063176155090332, + "policy/entropy_avg": 0.14582359790802002, + "step": 897, + "val/clipfrac_avg": 0.06062069162726402, + "val/num_eos_tokens": 1182, + "val/ratio": 0.9903557300567627, + "val/ratio_var": 5.06751166540198e-05 + }, + { + "episode": 14368, + "epoch": 0.7256565656565657, + "eps": 0, + "loss/policy_avg": 0.011427253484725952, + "loss/value_avg": 0.263162761926651, + "lr": 1.9350425816839875e-06, + "objective/entropy": -110.30941772460938, + "objective/kl": 41.13695526123047, + "objective/non_score_reward": -2.0568478107452393, + "objective/rlhf_reward": 13.72440242767334, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0011626393534243107, + "policy/clipfrac_avg": 0.006754214409738779, + "policy/entropy_avg": 0.2226581573486328, + "step": 898, + "val/clipfrac_avg": 0.0, + "val/num_eos_tokens": 10, + "val/ratio": 0.9850178956985474, + "val/ratio_var": 8.876781794242561e-05 + }, + { + "episode": 14384, + "epoch": 0.7264646464646465, + "eps": 0, + "loss/policy_avg": 0.11541193723678589, + "loss/value_avg": 0.4631091356277466, + "lr": 1.9245002169226814e-06, + "objective/entropy": -101.70489501953125, + "objective/kl": 48.557960510253906, + "objective/non_score_reward": -2.4278979301452637, + "objective/rlhf_reward": 13.314289093017578, + "objective/scores": 15.75, + "policy/approxkl_avg": 0.0009826980531215668, + "policy/clipfrac_avg": 0.004610124975442886, + "policy/entropy_avg": 0.1724681854248047, + "step": 899, + "val/clipfrac_avg": 0.013620924204587936, + "val/num_eos_tokens": 516, + "val/ratio": 0.9876973628997803, + "val/ratio_var": 5.4934793297434226e-05 + }, + { + "episode": 14400, + "epoch": 0.7272727272727273, + "eps": 0, + "loss/policy_avg": 0.07936800271272659, + "loss/value_avg": 0.6616516709327698, + "lr": 1.913979800444021e-06, + "objective/entropy": -91.96019744873047, + "objective/kl": 41.26939010620117, + "objective/non_score_reward": -2.063469409942627, + "objective/rlhf_reward": 12.522467613220215, + "objective/scores": 14.5625, + "policy/approxkl_avg": 0.0009332017507404089, + "policy/clipfrac_avg": 0.0043076397851109505, + "policy/entropy_avg": 0.18123245239257812, + "step": 900, + "val/clipfrac_avg": 0.01476163137704134, + "val/num_eos_tokens": 595, + "val/ratio": 0.9875973463058472, + "val/ratio_var": 7.475823076674715e-05 + } + ], + "logging_steps": 5, + "max_steps": 1238, + "num_input_tokens_seen": 0, + "num_train_epochs": 1.0, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": true, + "should_log": true, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0, + "train_batch_size": null, + "trial_name": null, + "trial_params": null +} diff --git a/ppo_dir_v2/v0-20260524-184726/checkpoint-900/training_args.bin b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f848d986c03a0e5da1d8a2b51d5cefaafe1d794f --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/checkpoint-900/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943608d919ddc1cb3922a234b0361623644bd5495f56c069ab6404575fbab56 +size 9169 diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_episode.png b/ppo_dir_v2/v0-20260524-184726/images/train_episode.png new file mode 100644 index 0000000000000000000000000000000000000000..d3894cb6e6c7d18ac16be289def6de6ec8bd8bc8 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_episode.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_epoch.png b/ppo_dir_v2/v0-20260524-184726/images/train_epoch.png new file mode 100644 index 0000000000000000000000000000000000000000..7b2b0498fbb4c3cfc297b9460add838d5a5208f6 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_epoch.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_eps.png b/ppo_dir_v2/v0-20260524-184726/images/train_eps.png new file mode 100644 index 0000000000000000000000000000000000000000..52d6d9e47cbfc88ad8211b1ee732ba84ecf98914 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_eps.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_loss_policy_avg.png b/ppo_dir_v2/v0-20260524-184726/images/train_loss_policy_avg.png new file mode 100644 index 0000000000000000000000000000000000000000..abee48c5f9ae7a13f445feb63f38ac12130c4d0c Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_loss_policy_avg.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_loss_value_avg.png b/ppo_dir_v2/v0-20260524-184726/images/train_loss_value_avg.png new file mode 100644 index 0000000000000000000000000000000000000000..733a17dd66a93d82412b011be2168e0da94a4407 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_loss_value_avg.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_lr.png b/ppo_dir_v2/v0-20260524-184726/images/train_lr.png new file mode 100644 index 0000000000000000000000000000000000000000..053017c3b7ccf4001883091856c1bccc7c1b9eef Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_lr.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_objective_entropy.png b/ppo_dir_v2/v0-20260524-184726/images/train_objective_entropy.png new file mode 100644 index 0000000000000000000000000000000000000000..42da855eac661d19fe7b720717e86d45d03f6dd1 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_objective_entropy.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_objective_kl.png b/ppo_dir_v2/v0-20260524-184726/images/train_objective_kl.png new file mode 100644 index 0000000000000000000000000000000000000000..e4b703ec971601e056c472e25d8ff3c25b3f156d Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_objective_kl.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_objective_non_score_reward.png b/ppo_dir_v2/v0-20260524-184726/images/train_objective_non_score_reward.png new file mode 100644 index 0000000000000000000000000000000000000000..fb711b329dbb2657e6742ec1c78efea64dba33b3 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_objective_non_score_reward.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_objective_rlhf_reward.png b/ppo_dir_v2/v0-20260524-184726/images/train_objective_rlhf_reward.png new file mode 100644 index 0000000000000000000000000000000000000000..57215980c343765f56dcb609f5ba8025cae68171 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_objective_rlhf_reward.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_objective_scores.png b/ppo_dir_v2/v0-20260524-184726/images/train_objective_scores.png new file mode 100644 index 0000000000000000000000000000000000000000..215f203d53cd6514a5eb18af5a0436491d38e81b Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_objective_scores.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_policy_approxkl_avg.png b/ppo_dir_v2/v0-20260524-184726/images/train_policy_approxkl_avg.png new file mode 100644 index 0000000000000000000000000000000000000000..4f6176b416d1a26acb0525df4227df3f0e138dd5 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_policy_approxkl_avg.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_policy_clipfrac_avg.png b/ppo_dir_v2/v0-20260524-184726/images/train_policy_clipfrac_avg.png new file mode 100644 index 0000000000000000000000000000000000000000..19561eb1945a4d178b673d3bd96b3209f420f00b Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_policy_clipfrac_avg.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_policy_entropy_avg.png b/ppo_dir_v2/v0-20260524-184726/images/train_policy_entropy_avg.png new file mode 100644 index 0000000000000000000000000000000000000000..2c9612228651ca611ec08d6077a51ec4d96be962 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_policy_entropy_avg.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_val_clipfrac_avg.png b/ppo_dir_v2/v0-20260524-184726/images/train_val_clipfrac_avg.png new file mode 100644 index 0000000000000000000000000000000000000000..990ddf612aa6bbb7826dafdb0dd96f400bc7150e Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_val_clipfrac_avg.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_val_num_eos_tokens.png b/ppo_dir_v2/v0-20260524-184726/images/train_val_num_eos_tokens.png new file mode 100644 index 0000000000000000000000000000000000000000..b06ccf2b74805354985cdceacad4fcfeaf7c63e1 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_val_num_eos_tokens.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_val_ratio.png b/ppo_dir_v2/v0-20260524-184726/images/train_val_ratio.png new file mode 100644 index 0000000000000000000000000000000000000000..442648809b64e63529c29c00ebf4d3639be41b4e Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_val_ratio.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/images/train_val_ratio_var.png b/ppo_dir_v2/v0-20260524-184726/images/train_val_ratio_var.png new file mode 100644 index 0000000000000000000000000000000000000000..0d9f99c96e46cc231c5ed0951f8ead716b2b50d0 Binary files /dev/null and b/ppo_dir_v2/v0-20260524-184726/images/train_val_ratio_var.png differ diff --git a/ppo_dir_v2/v0-20260524-184726/logging.jsonl b/ppo_dir_v2/v0-20260524-184726/logging.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ee306c02328a487095cfab5d081ea33c0fa22e3a --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/logging.jsonl @@ -0,0 +1,1239 @@ +{"eps": 0, "objective/kl": 4.4342165, "objective/entropy": -74.55732727, "objective/non_score_reward": -0.22171086, "objective/rlhf_reward": 9.18844509, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00091559, "policy/clipfrac_avg": 0.00112417, "loss/policy_avg": -0.2878437, "loss/value_avg": 10.60638714, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19521332, "val/ratio": 0.98925215, "val/ratio_var": 5.706e-05, "val/num_eos_tokens": 545, "lr": 0.0, "episode": 16, "epoch": 0.00080808, "global_step/max_steps": "1/1238", "percentage": "0.08%", "elapsed_time": "34s", "remaining_time": "11h 57m 39s"} +{"eps": 0, "objective/kl": 8.24789524, "objective/entropy": -73.15070343, "objective/non_score_reward": -0.41239476, "objective/rlhf_reward": 8.62666798, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0011616, "policy/clipfrac_avg": 0.0034807, "loss/policy_avg": -0.23873866, "loss/value_avg": 8.19930744, "val/clipfrac_avg": 0.00031834, "policy/entropy_avg": 0.26854444, "val/ratio": 0.98438537, "val/ratio_var": 0.00014293, "val/num_eos_tokens": 16, "lr": 1.6e-07, "episode": 32, "epoch": 0.00161616, "global_step/max_steps": "2/1238", "percentage": "0.16%", "elapsed_time": "1m 51s", "remaining_time": "19h 8m 52s"} +{"eps": 0, "objective/kl": 5.7903862, "objective/entropy": -57.60097885, "objective/non_score_reward": -0.28951931, "objective/rlhf_reward": 7.066926, "objective/scores": 7.34375, "policy/approxkl_avg": 0.00099819, "policy/clipfrac_avg": 0.00384272, "loss/policy_avg": -0.17912136, "loss/value_avg": 9.05179024, "val/clipfrac_avg": 0.00032502, "policy/entropy_avg": 0.2743988, "val/ratio": 0.98596704, "val/ratio_var": 7.311e-05, "val/num_eos_tokens": 85, "lr": 3.2e-07, "episode": 48, "epoch": 0.00242424, "global_step/max_steps": "3/1238", "percentage": "0.24%", "elapsed_time": "2m 23s", "remaining_time": "16h 21m 16s"} +{"eps": 0, "objective/kl": 8.57608986, "objective/entropy": -63.22553635, "objective/non_score_reward": -0.42880452, "objective/rlhf_reward": 7.74307013, "objective/scores": 8.1875, "policy/approxkl_avg": 0.00105408, "policy/clipfrac_avg": 0.00360234, "loss/policy_avg": -0.11512397, "loss/value_avg": 6.80494595, "val/clipfrac_avg": 0.00086094, "policy/entropy_avg": 0.24641418, "val/ratio": 0.98463726, "val/ratio_var": 7.188e-05, "val/num_eos_tokens": 422, "lr": 4.8e-07, "episode": 64, "epoch": 0.00323232, "global_step/max_steps": "4/1238", "percentage": "0.32%", "elapsed_time": "3m 7s", "remaining_time": "16h 1m 50s"} +{"eps": 0, "objective/kl": 5.36291981, "objective/entropy": -114.28401184, "objective/non_score_reward": -0.26814601, "objective/rlhf_reward": 7.38810396, "objective/scores": 7.65625, "policy/approxkl_avg": 0.00074071, "policy/clipfrac_avg": 0.0044956, "loss/policy_avg": -0.07829581, "loss/value_avg": 6.95508432, "val/clipfrac_avg": 0.00087029, "policy/entropy_avg": 0.16756058, "val/ratio": 0.98892975, "val/ratio_var": 4.612e-05, "val/num_eos_tokens": 962, "lr": 6.5e-07, "episode": 80, "epoch": 0.0040404, "global_step/max_steps": "5/1238", "percentage": "0.40%", "elapsed_time": "4m 15s", "remaining_time": "17h 31m 8s"} +{"eps": 0, "objective/kl": 5.35301304, "objective/entropy": -120.77016449, "objective/non_score_reward": -0.26765066, "objective/rlhf_reward": 8.77531815, "objective/scores": 9.0625, "policy/approxkl_avg": 0.00078115, "policy/clipfrac_avg": 0.00253526, "loss/policy_avg": -0.27658564, "loss/value_avg": 9.97632217, "val/clipfrac_avg": 0.00108166, "policy/entropy_avg": 0.21323395, "val/ratio": 0.98848772, "val/ratio_var": 6.569e-05, "val/num_eos_tokens": 128, "lr": 8.1e-07, "episode": 96, "epoch": 0.00484848, "global_step/max_steps": "6/1238", "percentage": "0.48%", "elapsed_time": "5m 28s", "remaining_time": "18h 42m 36s"} +{"eps": 0, "objective/kl": 6.92962313, "objective/entropy": -57.45833969, "objective/non_score_reward": -0.34648114, "objective/rlhf_reward": 7.0949254, "objective/scores": 7.4375, "policy/approxkl_avg": 0.00094215, "policy/clipfrac_avg": 0.00409944, "loss/policy_avg": -0.03681543, "loss/value_avg": 5.04795885, "val/clipfrac_avg": 0.00086028, "policy/entropy_avg": 0.20614624, "val/ratio": 0.98697793, "val/ratio_var": 3.151e-05, "val/num_eos_tokens": 150, "lr": 9.7e-07, "episode": 112, "epoch": 0.00565657, "global_step/max_steps": "7/1238", "percentage": "0.57%", "elapsed_time": "6m 11s", "remaining_time": "18h 8m 14s"} +{"eps": 0, "objective/kl": 4.52052021, "objective/entropy": -4.44236708, "objective/non_score_reward": -0.22602598, "objective/rlhf_reward": 11.03569221, "objective/scores": 11.25, "policy/approxkl_avg": 0.00110834, "policy/clipfrac_avg": 0.00427785, "loss/policy_avg": -0.13899218, "loss/value_avg": 12.85467911, "val/clipfrac_avg": 0.0012433, "policy/entropy_avg": 0.2364006, "val/ratio": 0.9858228, "val/ratio_var": 6.114e-05, "val/num_eos_tokens": 4, "lr": 1.13e-06, "episode": 128, "epoch": 0.00646465, "global_step/max_steps": "8/1238", "percentage": "0.65%", "elapsed_time": "6m 43s", "remaining_time": "17h 14m 29s"} +{"eps": 0, "objective/kl": 4.2430234, "objective/entropy": -90.2855835, "objective/non_score_reward": -0.21215117, "objective/rlhf_reward": 8.58863068, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00078025, "policy/clipfrac_avg": 0.00232383, "loss/policy_avg": -0.19064569, "loss/value_avg": 16.10247803, "val/clipfrac_avg": 0.00050829, "policy/entropy_avg": 0.16461068, "val/ratio": 0.98895007, "val/ratio_var": 5.19e-05, "val/num_eos_tokens": 11, "lr": 1.29e-06, "episode": 144, "epoch": 0.00727273, "global_step/max_steps": "9/1238", "percentage": "0.73%", "elapsed_time": "7m 28s", "remaining_time": "17h 0m 14s"} +{"eps": 0, "objective/kl": 8.20638084, "objective/entropy": -120.99843597, "objective/non_score_reward": -0.41031909, "objective/rlhf_reward": 6.39046192, "objective/scores": 6.8125, "policy/approxkl_avg": 0.00095779, "policy/clipfrac_avg": 0.00513067, "loss/policy_avg": -0.13336265, "loss/value_avg": 5.76433992, "val/clipfrac_avg": 0.0019273, "policy/entropy_avg": 0.23892975, "val/ratio": 0.98747623, "val/ratio_var": 6.364e-05, "val/num_eos_tokens": 368, "lr": 1.45e-06, "episode": 160, "epoch": 0.00808081, "global_step/max_steps": "10/1238", "percentage": "0.81%", "elapsed_time": "8m 20s", "remaining_time": "17h 4m 19s"} +{"eps": 0, "objective/kl": 5.86386156, "objective/entropy": -47.95066071, "objective/non_score_reward": -0.2931931, "objective/rlhf_reward": 8.07399464, "objective/scores": 8.375, "policy/approxkl_avg": 0.00115884, "policy/clipfrac_avg": 0.00382448, "loss/policy_avg": -0.1956511, "loss/value_avg": 9.05639458, "val/clipfrac_avg": 0.00221937, "policy/entropy_avg": 0.25668049, "val/ratio": 0.9839282, "val/ratio_var": 9.392e-05, "val/num_eos_tokens": 395, "lr": 1.61e-06, "episode": 176, "epoch": 0.00888889, "global_step/max_steps": "11/1238", "percentage": "0.89%", "elapsed_time": "8m 51s", "remaining_time": "16h 27m 49s"} +{"eps": 0, "objective/kl": 7.08340883, "objective/entropy": -56.2935791, "objective/non_score_reward": -0.35417044, "objective/rlhf_reward": 8.0833292, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00103084, "policy/clipfrac_avg": 0.00467706, "loss/policy_avg": -0.09844053, "loss/value_avg": 5.99802637, "val/clipfrac_avg": 0.00109912, "policy/entropy_avg": 0.21318722, "val/ratio": 0.98597157, "val/ratio_var": 4.266e-05, "val/num_eos_tokens": 36, "lr": 1.77e-06, "episode": 192, "epoch": 0.00969697, "global_step/max_steps": "12/1238", "percentage": "0.97%", "elapsed_time": "9m 51s", "remaining_time": "16h 46m 58s"} +{"eps": 0, "objective/kl": 6.38713884, "objective/entropy": -47.07265472, "objective/non_score_reward": -0.31935692, "objective/rlhf_reward": 6.65720558, "objective/scores": 6.96875, "policy/approxkl_avg": 0.00116117, "policy/clipfrac_avg": 0.0056226, "loss/policy_avg": -0.1042285, "loss/value_avg": 7.21542406, "val/clipfrac_avg": 0.00599584, "policy/entropy_avg": 0.25177002, "val/ratio": 0.98470861, "val/ratio_var": 6.943e-05, "val/num_eos_tokens": 541, "lr": 1.94e-06, "episode": 208, "epoch": 0.01050505, "global_step/max_steps": "13/1238", "percentage": "1.05%", "elapsed_time": "10m 30s", "remaining_time": "16h 30m 41s"} +{"eps": 0, "objective/kl": 6.54490757, "objective/entropy": -41.00790405, "objective/non_score_reward": -0.32724538, "objective/rlhf_reward": 8.28017616, "objective/scores": 8.625, "policy/approxkl_avg": 0.00108748, "policy/clipfrac_avg": 0.00600705, "loss/policy_avg": -0.12459716, "loss/value_avg": 7.58163309, "val/clipfrac_avg": 0.00567345, "policy/entropy_avg": 0.26541901, "val/ratio": 0.98575783, "val/ratio_var": 9.339e-05, "val/num_eos_tokens": 407, "lr": 2.1e-06, "episode": 224, "epoch": 0.01131313, "global_step/max_steps": "14/1238", "percentage": "1.13%", "elapsed_time": "11m 1s", "remaining_time": "16h 3m 30s"} +{"eps": 0, "objective/kl": 6.37682581, "objective/entropy": -152.62283325, "objective/non_score_reward": -0.31884128, "objective/rlhf_reward": 8.24756527, "objective/scores": 8.5625, "policy/approxkl_avg": 0.00080439, "policy/clipfrac_avg": 0.00408326, "loss/policy_avg": -0.35094249, "loss/value_avg": 8.65149307, "val/clipfrac_avg": 0.02940819, "policy/entropy_avg": 0.19263721, "val/ratio": 0.98883432, "val/ratio_var": 6.805e-05, "val/num_eos_tokens": 479, "lr": 2.26e-06, "episode": 240, "epoch": 0.01212121, "global_step/max_steps": "15/1238", "percentage": "1.21%", "elapsed_time": "11m 34s", "remaining_time": "15h 44m 6s"} +{"eps": 0, "objective/kl": 7.03504753, "objective/entropy": -69.35281372, "objective/non_score_reward": -0.3517524, "objective/rlhf_reward": 8.05059147, "objective/scores": 8.375, "policy/approxkl_avg": 0.00111618, "policy/clipfrac_avg": 0.00517032, "loss/policy_avg": -0.2047292, "loss/value_avg": 7.68352795, "val/clipfrac_avg": 0.05127308, "policy/entropy_avg": 0.24970913, "val/ratio": 0.98480487, "val/ratio_var": 8.986e-05, "val/num_eos_tokens": 17, "lr": 2.42e-06, "episode": 256, "epoch": 0.01292929, "global_step/max_steps": "16/1238", "percentage": "1.29%", "elapsed_time": "12m 25s", "remaining_time": "15h 49m 14s"} +{"eps": 0, "objective/kl": 6.51870966, "objective/entropy": -107.7979126, "objective/non_score_reward": -0.32593548, "objective/rlhf_reward": 8.93187714, "objective/scores": 9.25, "policy/approxkl_avg": 0.00094233, "policy/clipfrac_avg": 0.00356655, "loss/policy_avg": -0.32048213, "loss/value_avg": 13.41343689, "val/clipfrac_avg": 0.06655061, "policy/entropy_avg": 0.1995694, "val/ratio": 0.98727322, "val/ratio_var": 6.392e-05, "val/num_eos_tokens": 636, "lr": 2.58e-06, "episode": 272, "epoch": 0.01373737, "global_step/max_steps": "17/1238", "percentage": "1.37%", "elapsed_time": "13m 16s", "remaining_time": "15h 54m 3s"} +{"eps": 0, "objective/kl": 3.65440035, "objective/entropy": -60.37420654, "objective/non_score_reward": -0.18272004, "objective/rlhf_reward": 7.47060061, "objective/scores": 7.65625, "policy/approxkl_avg": 0.00091095, "policy/clipfrac_avg": 0.00324248, "loss/policy_avg": -0.16326521, "loss/value_avg": 7.02220631, "val/clipfrac_avg": 0.15316108, "policy/entropy_avg": 0.21929169, "val/ratio": 0.98750043, "val/ratio_var": 6.944e-05, "val/num_eos_tokens": 376, "lr": 2.74e-06, "episode": 288, "epoch": 0.01454545, "global_step/max_steps": "18/1238", "percentage": "1.45%", "elapsed_time": "13m 44s", "remaining_time": "15h 31m 21s"} +{"eps": 0, "objective/kl": 8.1903553, "objective/entropy": -64.48834229, "objective/non_score_reward": -0.40951771, "objective/rlhf_reward": 7.08267021, "objective/scores": 7.5, "policy/approxkl_avg": 0.00096292, "policy/clipfrac_avg": 0.00442806, "loss/policy_avg": -0.09499571, "loss/value_avg": 4.84756756, "val/clipfrac_avg": 0.09573162, "policy/entropy_avg": 0.21021271, "val/ratio": 0.98718858, "val/ratio_var": 5.79e-05, "val/num_eos_tokens": 161, "lr": 2.9e-06, "episode": 304, "epoch": 0.01535354, "global_step/max_steps": "19/1238", "percentage": "1.53%", "elapsed_time": "14m 31s", "remaining_time": "15h 31m 40s"} +{"eps": 0, "objective/kl": 4.76213169, "objective/entropy": -92.24372864, "objective/non_score_reward": -0.23810658, "objective/rlhf_reward": 8.15056515, "objective/scores": 8.375, "policy/approxkl_avg": 0.00097188, "policy/clipfrac_avg": 0.00447654, "loss/policy_avg": -0.3619737, "loss/value_avg": 9.99121094, "val/clipfrac_avg": 0.15394406, "policy/entropy_avg": 0.26179969, "val/ratio": 0.98751813, "val/ratio_var": 9.374e-05, "val/num_eos_tokens": 306, "lr": 3.06e-06, "episode": 320, "epoch": 0.01616162, "global_step/max_steps": "20/1238", "percentage": "1.62%", "elapsed_time": "15m 2s", "remaining_time": "15h 16m 30s"} +{"eps": 0, "objective/kl": 5.37165451, "objective/entropy": -60.66798401, "objective/non_score_reward": -0.2685827, "objective/rlhf_reward": 6.95797968, "objective/scores": 7.21875, "policy/approxkl_avg": 0.00097205, "policy/clipfrac_avg": 0.00218608, "loss/policy_avg": -0.0626925, "loss/value_avg": 5.03910637, "val/clipfrac_avg": 0.11872587, "policy/entropy_avg": 0.23161316, "val/ratio": 0.98667836, "val/ratio_var": 5.08e-05, "val/num_eos_tokens": 62, "lr": 3.23e-06, "episode": 336, "epoch": 0.0169697, "global_step/max_steps": "21/1238", "percentage": "1.70%", "elapsed_time": "15m 48s", "remaining_time": "15h 16m 26s"} +{"eps": 0, "objective/kl": 3.48607635, "objective/entropy": -52.2295723, "objective/non_score_reward": -0.17430381, "objective/rlhf_reward": 8.28663445, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00078505, "policy/clipfrac_avg": 0.00418407, "loss/policy_avg": -0.17756629, "loss/value_avg": 9.87999535, "val/clipfrac_avg": 0.19386646, "policy/entropy_avg": 0.17317486, "val/ratio": 0.98837137, "val/ratio_var": 5.719e-05, "val/num_eos_tokens": 227, "lr": 3.39e-06, "episode": 352, "epoch": 0.01777778, "global_step/max_steps": "22/1238", "percentage": "1.78%", "elapsed_time": "16m 18s", "remaining_time": "15h 1m 11s"} +{"eps": 0, "objective/kl": 6.3290472, "objective/entropy": -17.84231758, "objective/non_score_reward": -0.31645238, "objective/rlhf_reward": 9.14448547, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00114364, "policy/clipfrac_avg": 0.00468597, "loss/policy_avg": -0.0860893, "loss/value_avg": 5.0266819, "val/clipfrac_avg": 0.11751493, "policy/entropy_avg": 0.24533081, "val/ratio": 0.9848727, "val/ratio_var": 6.24e-05, "val/num_eos_tokens": 412, "lr": 3.55e-06, "episode": 368, "epoch": 0.01858586, "global_step/max_steps": "23/1238", "percentage": "1.86%", "elapsed_time": "16m 55s", "remaining_time": "14h 53m 56s"} +{"eps": 0, "objective/kl": 5.16103458, "objective/entropy": -47.78543091, "objective/non_score_reward": -0.25805175, "objective/rlhf_reward": 7.45288563, "objective/scores": 7.71875, "policy/approxkl_avg": 0.00094654, "policy/clipfrac_avg": 0.00397715, "loss/policy_avg": -0.01079767, "loss/value_avg": 6.27596188, "val/clipfrac_avg": 0.12968723, "policy/entropy_avg": 0.28936768, "val/ratio": 0.9866119, "val/ratio_var": 6.631e-05, "val/num_eos_tokens": 200, "lr": 3.71e-06, "episode": 384, "epoch": 0.01939394, "global_step/max_steps": "24/1238", "percentage": "1.94%", "elapsed_time": "17m 26s", "remaining_time": "14h 42m 24s"} +{"eps": 0, "objective/kl": 7.75646591, "objective/entropy": -20.86711121, "objective/non_score_reward": -0.38782334, "objective/rlhf_reward": 7.92077065, "objective/scores": 8.3125, "policy/approxkl_avg": 0.00122294, "policy/clipfrac_avg": 0.00300888, "loss/policy_avg": 0.03891272, "loss/value_avg": 5.36677265, "val/clipfrac_avg": 0.12619257, "policy/entropy_avg": 0.25541687, "val/ratio": 0.98582876, "val/ratio_var": 6.28e-05, "val/num_eos_tokens": 16, "lr": 3.87e-06, "episode": 400, "epoch": 0.02020202, "global_step/max_steps": "25/1238", "percentage": "2.02%", "elapsed_time": "18m 18s", "remaining_time": "14h 48m 38s"} +{"eps": 0, "objective/kl": 6.12144804, "objective/entropy": -131.188797, "objective/non_score_reward": -0.30607241, "objective/rlhf_reward": 7.20174026, "objective/scores": 7.5, "policy/approxkl_avg": 0.00086304, "policy/clipfrac_avg": 0.00351269, "loss/policy_avg": -0.09346848, "loss/value_avg": 3.79160166, "val/clipfrac_avg": 0.1054671, "policy/entropy_avg": 0.21083069, "val/ratio": 0.98748004, "val/ratio_var": 6.414e-05, "val/num_eos_tokens": 212, "lr": 4.03e-06, "episode": 416, "epoch": 0.0210101, "global_step/max_steps": "26/1238", "percentage": "2.10%", "elapsed_time": "19m 5s", "remaining_time": "14h 49m 49s"} +{"eps": 0, "objective/kl": 6.76476336, "objective/entropy": -91.82192993, "objective/non_score_reward": -0.33823818, "objective/rlhf_reward": 9.75160599, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00094778, "policy/clipfrac_avg": 0.00545152, "loss/policy_avg": -0.08439042, "loss/value_avg": 4.86584044, "val/clipfrac_avg": 0.06509674, "policy/entropy_avg": 0.20922852, "val/ratio": 0.98702848, "val/ratio_var": 4.724e-05, "val/num_eos_tokens": 221, "lr": 4.19e-06, "episode": 432, "epoch": 0.02181818, "global_step/max_steps": "27/1238", "percentage": "2.18%", "elapsed_time": "19m 42s", "remaining_time": "14h 43m 39s"} +{"eps": 0, "objective/kl": 7.10296059, "objective/entropy": -68.85742188, "objective/non_score_reward": -0.35514808, "objective/rlhf_reward": 8.93782043, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00103954, "policy/clipfrac_avg": 0.00519603, "loss/policy_avg": -0.12459178, "loss/value_avg": 4.97368526, "val/clipfrac_avg": 0.09726892, "policy/entropy_avg": 0.22506142, "val/ratio": 0.98593259, "val/ratio_var": 7.373e-05, "val/num_eos_tokens": 153, "lr": 4.35e-06, "episode": 448, "epoch": 0.02262626, "global_step/max_steps": "28/1238", "percentage": "2.26%", "elapsed_time": "20m 17s", "remaining_time": "14h 36m 43s"} +{"eps": 0, "objective/kl": 4.58580685, "objective/entropy": -39.28635788, "objective/non_score_reward": -0.22929037, "objective/rlhf_reward": 7.24336576, "objective/scores": 7.46875, "policy/approxkl_avg": 0.00107841, "policy/clipfrac_avg": 0.00465692, "loss/policy_avg": -0.0893494, "loss/value_avg": 4.28043509, "val/clipfrac_avg": 0.03393328, "policy/entropy_avg": 0.28299713, "val/ratio": 0.98540258, "val/ratio_var": 7.98e-05, "val/num_eos_tokens": 151, "lr": 4.52e-06, "episode": 464, "epoch": 0.02343434, "global_step/max_steps": "29/1238", "percentage": "2.34%", "elapsed_time": "20m 52s", "remaining_time": "14h 30m 18s"} +{"eps": 0, "objective/kl": 3.25834656, "objective/entropy": -56.1607933, "objective/non_score_reward": -0.16291733, "objective/rlhf_reward": 6.47184849, "objective/scores": 6.625, "policy/approxkl_avg": 0.00090388, "policy/clipfrac_avg": 0.00398654, "loss/policy_avg": 0.00096748, "loss/value_avg": 3.29168797, "val/clipfrac_avg": 0.0377676, "policy/entropy_avg": 0.21944809, "val/ratio": 0.98807073, "val/ratio_var": 7.167e-05, "val/num_eos_tokens": 229, "lr": 4.68e-06, "episode": 480, "epoch": 0.02424242, "global_step/max_steps": "30/1238", "percentage": "2.42%", "elapsed_time": "21m 18s", "remaining_time": "14h 18m 18s"} +{"eps": 0, "objective/kl": 4.62398624, "objective/entropy": -85.58920288, "objective/non_score_reward": -0.23119934, "objective/rlhf_reward": 8.10473824, "objective/scores": 8.3125, "policy/approxkl_avg": 0.00082341, "policy/clipfrac_avg": 0.00375101, "loss/policy_avg": -0.05671697, "loss/value_avg": 4.55543995, "val/clipfrac_avg": 0.12869637, "policy/entropy_avg": 0.18961239, "val/ratio": 0.9884215, "val/ratio_var": 7.101e-05, "val/num_eos_tokens": 30, "lr": 4.84e-06, "episode": 496, "epoch": 0.02505051, "global_step/max_steps": "31/1238", "percentage": "2.50%", "elapsed_time": "21m 53s", "remaining_time": "14h 12m 27s"} +{"eps": 0, "objective/kl": 5.96487904, "objective/entropy": -81.65757751, "objective/non_score_reward": -0.29824397, "objective/rlhf_reward": 6.811131, "objective/scores": 7.125, "policy/approxkl_avg": 0.00081651, "policy/clipfrac_avg": 0.00192898, "loss/policy_avg": -0.08824437, "loss/value_avg": 3.42183352, "val/clipfrac_avg": 0.10708418, "policy/entropy_avg": 0.19529819, "val/ratio": 0.98819721, "val/ratio_var": 7.151e-05, "val/num_eos_tokens": 82, "lr": 5e-06, "episode": 512, "epoch": 0.02585859, "global_step/max_steps": "32/1238", "percentage": "2.58%", "elapsed_time": "22m 29s", "remaining_time": "14h 7m 35s"} +{"eps": 0, "objective/kl": 3.57955456, "objective/entropy": -73.17301178, "objective/non_score_reward": -0.17897773, "objective/rlhf_reward": 7.76633453, "objective/scores": 7.9375, "policy/approxkl_avg": 0.00083787, "policy/clipfrac_avg": 0.00224451, "loss/policy_avg": 0.09306782, "loss/value_avg": 3.89824343, "val/clipfrac_avg": 0.09437251, "policy/entropy_avg": 0.22289753, "val/ratio": 0.98604202, "val/ratio_var": 6.518e-05, "val/num_eos_tokens": 388, "lr": 5.16e-06, "episode": 528, "epoch": 0.02666667, "global_step/max_steps": "33/1238", "percentage": "2.67%", "elapsed_time": "22m 57s", "remaining_time": "13h 58m 33s"} +{"eps": 0, "objective/kl": 5.92071533, "objective/entropy": -104.75776672, "objective/non_score_reward": -0.2960358, "objective/rlhf_reward": 8.38755798, "objective/scores": 8.6875, "policy/approxkl_avg": 0.00101151, "policy/clipfrac_avg": 0.00496835, "loss/policy_avg": -0.23503622, "loss/value_avg": 5.29703331, "val/clipfrac_avg": 0.33360282, "policy/entropy_avg": 0.22486305, "val/ratio": 0.98697263, "val/ratio_var": 0.00013502, "val/num_eos_tokens": 871, "lr": 5.32e-06, "episode": 544, "epoch": 0.02747475, "global_step/max_steps": "34/1238", "percentage": "2.75%", "elapsed_time": "23m 46s", "remaining_time": "14h 1m 41s"} +{"eps": 0, "objective/kl": 5.67402267, "objective/entropy": -92.61628723, "objective/non_score_reward": -0.28370115, "objective/rlhf_reward": 6.54442406, "objective/scores": 6.8125, "policy/approxkl_avg": 0.00091261, "policy/clipfrac_avg": 0.00544425, "loss/policy_avg": 0.01827317, "loss/value_avg": 2.6717968, "val/clipfrac_avg": 0.19933006, "policy/entropy_avg": 0.28992081, "val/ratio": 0.98733664, "val/ratio_var": 0.00010929, "val/num_eos_tokens": 494, "lr": 5.48e-06, "episode": 560, "epoch": 0.02828283, "global_step/max_steps": "35/1238", "percentage": "2.83%", "elapsed_time": "24m 24s", "remaining_time": "13h 58m 45s"} +{"eps": 0, "objective/kl": 5.0494523, "objective/entropy": -116.77957916, "objective/non_score_reward": -0.25247261, "objective/rlhf_reward": 11.34127808, "objective/scores": 11.625, "policy/approxkl_avg": 0.00078754, "policy/clipfrac_avg": 0.00224344, "loss/policy_avg": -0.45309979, "loss/value_avg": 7.70558357, "val/clipfrac_avg": 0.41723159, "policy/entropy_avg": 0.16874886, "val/ratio": 0.98910254, "val/ratio_var": 7.57e-05, "val/num_eos_tokens": 472, "lr": 5.65e-06, "episode": 576, "epoch": 0.02909091, "global_step/max_steps": "36/1238", "percentage": "2.91%", "elapsed_time": "24m 55s", "remaining_time": "13h 52m 16s"} +{"eps": 0, "objective/kl": 6.88625383, "objective/entropy": -141.64932251, "objective/non_score_reward": -0.34431267, "objective/rlhf_reward": 8.91349983, "objective/scores": 9.25, "policy/approxkl_avg": 0.00091498, "policy/clipfrac_avg": 0.00364487, "loss/policy_avg": -0.27262574, "loss/value_avg": 4.82029629, "val/clipfrac_avg": 0.12755424, "policy/entropy_avg": 0.20346355, "val/ratio": 0.98800653, "val/ratio_var": 0.00010918, "val/num_eos_tokens": 163, "lr": 5.81e-06, "episode": 592, "epoch": 0.02989899, "global_step/max_steps": "37/1238", "percentage": "2.99%", "elapsed_time": "25m 41s", "remaining_time": "13h 54m 4s"} +{"eps": 0, "objective/kl": 4.56805897, "objective/entropy": -68.97830963, "objective/non_score_reward": -0.22840297, "objective/rlhf_reward": 8.59386253, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00097087, "policy/clipfrac_avg": 0.00377998, "loss/policy_avg": -0.34780562, "loss/value_avg": 6.75795174, "val/clipfrac_avg": 0.19302648, "policy/entropy_avg": 0.23283911, "val/ratio": 0.98636854, "val/ratio_var": 0.00010391, "val/num_eos_tokens": 504, "lr": 5.97e-06, "episode": 608, "epoch": 0.03070707, "global_step/max_steps": "38/1238", "percentage": "3.07%", "elapsed_time": "26m 10s", "remaining_time": "13h 46m 31s"} +{"eps": 0, "objective/kl": 4.91707182, "objective/entropy": -96.96963501, "objective/non_score_reward": -0.2458536, "objective/rlhf_reward": 8.47289562, "objective/scores": 8.75, "policy/approxkl_avg": 0.00084122, "policy/clipfrac_avg": 0.00687195, "loss/policy_avg": -0.19798313, "loss/value_avg": 3.17144871, "val/clipfrac_avg": 0.14115071, "policy/entropy_avg": 0.18240166, "val/ratio": 0.98875636, "val/ratio_var": 7.866e-05, "val/num_eos_tokens": 150, "lr": 6.13e-06, "episode": 624, "epoch": 0.03151515, "global_step/max_steps": "39/1238", "percentage": "3.15%", "elapsed_time": "26m 49s", "remaining_time": "13h 44m 51s"} +{"eps": 0, "objective/kl": 6.14642763, "objective/entropy": -71.3642807, "objective/non_score_reward": -0.30732143, "objective/rlhf_reward": 8.70049095, "objective/scores": 9.0, "policy/approxkl_avg": 0.00080698, "policy/clipfrac_avg": 0.00431099, "loss/policy_avg": -0.18319812, "loss/value_avg": 2.14117146, "val/clipfrac_avg": 0.1166086, "policy/entropy_avg": 0.15991306, "val/ratio": 0.98956978, "val/ratio_var": 3.527e-05, "val/num_eos_tokens": 20, "lr": 6.29e-06, "episode": 640, "epoch": 0.03232323, "global_step/max_steps": "40/1238", "percentage": "3.23%", "elapsed_time": "27m 27s", "remaining_time": "13h 42m 23s"} +{"eps": 0, "objective/kl": 6.44523478, "objective/entropy": -72.12484741, "objective/non_score_reward": -0.32226175, "objective/rlhf_reward": 9.90430069, "objective/scores": 10.25, "policy/approxkl_avg": 0.00110115, "policy/clipfrac_avg": 0.00589251, "loss/policy_avg": -0.05084063, "loss/value_avg": 3.10752487, "val/clipfrac_avg": 0.28902361, "policy/entropy_avg": 0.23779297, "val/ratio": 0.98578787, "val/ratio_var": 8.489e-05, "val/num_eos_tokens": 54, "lr": 6.45e-06, "episode": 656, "epoch": 0.03313131, "global_step/max_steps": "41/1238", "percentage": "3.31%", "elapsed_time": "28m 16s", "remaining_time": "13h 45m 22s"} +{"eps": 0, "objective/kl": 7.49373245, "objective/entropy": -90.39147186, "objective/non_score_reward": -0.37468663, "objective/rlhf_reward": 6.71076298, "objective/scores": 7.09375, "policy/approxkl_avg": 0.00082061, "policy/clipfrac_avg": 0.00267806, "loss/policy_avg": -0.12600231, "loss/value_avg": 4.62621498, "val/clipfrac_avg": 0.11169028, "policy/entropy_avg": 0.19233227, "val/ratio": 0.98852193, "val/ratio_var": 8.111e-05, "val/num_eos_tokens": 8, "lr": 6.61e-06, "episode": 672, "epoch": 0.03393939, "global_step/max_steps": "42/1238", "percentage": "3.39%", "elapsed_time": "29m 8s", "remaining_time": "13h 49m 41s"} +{"eps": 0, "objective/kl": 5.22401333, "objective/entropy": -97.91592407, "objective/non_score_reward": -0.2612007, "objective/rlhf_reward": 9.0630188, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00108776, "policy/clipfrac_avg": 0.00464998, "loss/policy_avg": -0.43953514, "loss/value_avg": 6.37335491, "val/clipfrac_avg": 0.22453558, "policy/entropy_avg": 0.20863533, "val/ratio": 0.98562038, "val/ratio_var": 0.00013598, "val/num_eos_tokens": 302, "lr": 6.77e-06, "episode": 688, "epoch": 0.03474747, "global_step/max_steps": "43/1238", "percentage": "3.47%", "elapsed_time": "29m 47s", "remaining_time": "13h 48m 8s"} +{"eps": 0, "objective/kl": 8.19976616, "objective/entropy": -113.14868164, "objective/non_score_reward": -0.40998834, "objective/rlhf_reward": 6.9806366, "objective/scores": 7.375, "policy/approxkl_avg": 0.00094785, "policy/clipfrac_avg": 0.00245037, "loss/policy_avg": -0.2054598, "loss/value_avg": 3.10078382, "val/clipfrac_avg": 0.1179712, "policy/entropy_avg": 0.21928471, "val/ratio": 0.98704666, "val/ratio_var": 9.354e-05, "val/num_eos_tokens": 408, "lr": 6.94e-06, "episode": 704, "epoch": 0.03555556, "global_step/max_steps": "44/1238", "percentage": "3.55%", "elapsed_time": "30m 27s", "remaining_time": "13h 46m 19s"} +{"eps": 0, "objective/kl": 6.22565937, "objective/entropy": -61.80511475, "objective/non_score_reward": -0.31128299, "objective/rlhf_reward": 7.52465439, "objective/scores": 7.84375, "policy/approxkl_avg": 0.00094083, "policy/clipfrac_avg": 0.00496809, "loss/policy_avg": 0.06887192, "loss/value_avg": 1.43515038, "val/clipfrac_avg": 0.17404783, "policy/entropy_avg": 0.19115067, "val/ratio": 0.98862481, "val/ratio_var": 4.797e-05, "val/num_eos_tokens": 165, "lr": 7.1e-06, "episode": 720, "epoch": 0.03636364, "global_step/max_steps": "45/1238", "percentage": "3.63%", "elapsed_time": "31m 20s", "remaining_time": "13h 50m 46s"} +{"eps": 0, "objective/kl": 9.15308571, "objective/entropy": -80.87551117, "objective/non_score_reward": -0.45765427, "objective/rlhf_reward": 7.3939085, "objective/scores": 7.84375, "policy/approxkl_avg": 0.00097026, "policy/clipfrac_avg": 0.00387326, "loss/policy_avg": -0.00872729, "loss/value_avg": 1.20116162, "val/clipfrac_avg": 0.15191919, "policy/entropy_avg": 0.20413971, "val/ratio": 0.98642582, "val/ratio_var": 5.903e-05, "val/num_eos_tokens": 330, "lr": 7.26e-06, "episode": 736, "epoch": 0.03717172, "global_step/max_steps": "46/1238", "percentage": "3.72%", "elapsed_time": "32m 2s", "remaining_time": "13h 50m 28s"} +{"eps": 0, "objective/kl": 7.73715687, "objective/entropy": -74.29957581, "objective/non_score_reward": -0.38685787, "objective/rlhf_reward": 7.06626701, "objective/scores": 7.4375, "policy/approxkl_avg": 0.00089439, "policy/clipfrac_avg": 0.00720657, "loss/policy_avg": -0.00994584, "loss/value_avg": 1.08387923, "val/clipfrac_avg": 0.15996163, "policy/entropy_avg": 0.1891346, "val/ratio": 0.98792422, "val/ratio_var": 4.615e-05, "val/num_eos_tokens": 539, "lr": 7.42e-06, "episode": 752, "epoch": 0.0379798, "global_step/max_steps": "47/1238", "percentage": "3.80%", "elapsed_time": "32m 40s", "remaining_time": "13h 48m 0s"} +{"eps": 0, "objective/kl": 4.25390482, "objective/entropy": -73.60452271, "objective/non_score_reward": -0.21269527, "objective/rlhf_reward": 7.46503925, "objective/scores": 7.6875, "policy/approxkl_avg": 0.00101192, "policy/clipfrac_avg": 0.01086447, "loss/policy_avg": -0.12174442, "loss/value_avg": 4.8970499, "val/clipfrac_avg": 0.27839154, "policy/entropy_avg": 0.25343132, "val/ratio": 0.98669982, "val/ratio_var": 0.00011045, "val/num_eos_tokens": 180, "lr": 7.58e-06, "episode": 768, "epoch": 0.03878788, "global_step/max_steps": "48/1238", "percentage": "3.88%", "elapsed_time": "33m 5s", "remaining_time": "13h 40m 24s"} +{"eps": 0, "objective/kl": 6.29369354, "objective/entropy": -61.9909935, "objective/non_score_reward": -0.31468469, "objective/rlhf_reward": 7.89234638, "objective/scores": 8.1875, "policy/approxkl_avg": 0.00107941, "policy/clipfrac_avg": 0.01386506, "loss/policy_avg": -0.39860064, "loss/value_avg": 3.1691649, "val/clipfrac_avg": 0.16114408, "policy/entropy_avg": 0.21884394, "val/ratio": 0.98582405, "val/ratio_var": 0.00017288, "val/num_eos_tokens": 13, "lr": 7.74e-06, "episode": 784, "epoch": 0.03959596, "global_step/max_steps": "49/1238", "percentage": "3.96%", "elapsed_time": "33m 41s", "remaining_time": "13h 37m 42s"} +{"eps": 0, "objective/kl": 6.16764307, "objective/entropy": -71.79003906, "objective/non_score_reward": -0.30838215, "objective/rlhf_reward": 7.57638359, "objective/scores": 7.875, "policy/approxkl_avg": 0.00096426, "policy/clipfrac_avg": 0.00900818, "loss/policy_avg": -0.01876144, "loss/value_avg": 1.69523811, "val/clipfrac_avg": 0.14082018, "policy/entropy_avg": 0.2461195, "val/ratio": 0.98826408, "val/ratio_var": 6.893e-05, "val/num_eos_tokens": 98, "lr": 7.9e-06, "episode": 800, "epoch": 0.04040404, "global_step/max_steps": "50/1238", "percentage": "4.04%", "elapsed_time": "34m 19s", "remaining_time": "13h 35m 28s"} +{"eps": 0, "objective/kl": 5.65581131, "objective/entropy": -79.89970398, "objective/non_score_reward": -0.28279057, "objective/rlhf_reward": 7.61174107, "objective/scores": 7.90625, "policy/approxkl_avg": 0.00088013, "policy/clipfrac_avg": 0.00721607, "loss/policy_avg": -0.04541177, "loss/value_avg": 1.30623686, "val/clipfrac_avg": 0.07814655, "policy/entropy_avg": 0.18111515, "val/ratio": 0.98852837, "val/ratio_var": 8.526e-05, "val/num_eos_tokens": 21, "lr": 8.06e-06, "episode": 816, "epoch": 0.04121212, "global_step/max_steps": "51/1238", "percentage": "4.12%", "elapsed_time": "34m 57s", "remaining_time": "13h 33m 28s"} +{"eps": 0, "objective/kl": 5.5109024, "objective/entropy": -170.48452759, "objective/non_score_reward": -0.27554512, "objective/rlhf_reward": 7.02963066, "objective/scores": 7.3125, "policy/approxkl_avg": 0.00061088, "policy/clipfrac_avg": 0.00431482, "loss/policy_avg": -0.14512324, "loss/value_avg": 1.53890049, "val/clipfrac_avg": 0.08058576, "policy/entropy_avg": 0.13148308, "val/ratio": 0.99192321, "val/ratio_var": 4.702e-05, "val/num_eos_tokens": 337, "lr": 8.23e-06, "episode": 832, "epoch": 0.0420202, "global_step/max_steps": "52/1238", "percentage": "4.20%", "elapsed_time": "35m 45s", "remaining_time": "13h 35m 24s"} +{"eps": 0, "objective/kl": 6.6281395, "objective/entropy": -96.16239166, "objective/non_score_reward": -0.33140695, "objective/rlhf_reward": 9.0787487, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00083596, "policy/clipfrac_avg": 0.00566166, "loss/policy_avg": -0.3828167, "loss/value_avg": 3.00696564, "val/clipfrac_avg": 0.10249013, "policy/entropy_avg": 0.17138624, "val/ratio": 0.98931181, "val/ratio_var": 6.531e-05, "val/num_eos_tokens": 9, "lr": 8.39e-06, "episode": 848, "epoch": 0.04282828, "global_step/max_steps": "53/1238", "percentage": "4.28%", "elapsed_time": "36m 22s", "remaining_time": "13h 33m 19s"} +{"eps": 0, "objective/kl": 8.29164696, "objective/entropy": -79.8026123, "objective/non_score_reward": -0.41458237, "objective/rlhf_reward": 9.0776062, "objective/scores": 9.5, "policy/approxkl_avg": 0.00094871, "policy/clipfrac_avg": 0.02798136, "loss/policy_avg": -0.0157621, "loss/value_avg": 1.70618403, "val/clipfrac_avg": 0.12049608, "policy/entropy_avg": 0.22313881, "val/ratio": 0.98761213, "val/ratio_var": 7.098e-05, "val/num_eos_tokens": 151, "lr": 8.55e-06, "episode": 864, "epoch": 0.04363636, "global_step/max_steps": "54/1238", "percentage": "4.36%", "elapsed_time": "37m 3s", "remaining_time": "13h 32m 35s"} +{"eps": 0, "objective/kl": 10.05303288, "objective/entropy": -34.68245697, "objective/non_score_reward": -0.50265169, "objective/rlhf_reward": 9.54031754, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00096774, "policy/clipfrac_avg": 0.00330346, "loss/policy_avg": -0.33922648, "loss/value_avg": 1.76912308, "val/clipfrac_avg": 0.17926592, "policy/entropy_avg": 0.19804573, "val/ratio": 0.98715234, "val/ratio_var": 4.744e-05, "val/num_eos_tokens": 45, "lr": 8.71e-06, "episode": 880, "epoch": 0.04444444, "global_step/max_steps": "55/1238", "percentage": "4.44%", "elapsed_time": "37m 40s", "remaining_time": "13h 30m 15s"} +{"eps": 0, "objective/kl": 7.36651039, "objective/entropy": -89.58358765, "objective/non_score_reward": -0.3683255, "objective/rlhf_reward": 7.39729929, "objective/scores": 7.75, "policy/approxkl_avg": 0.00086844, "policy/clipfrac_avg": 0.01221869, "loss/policy_avg": -0.14861116, "loss/value_avg": 3.01874971, "val/clipfrac_avg": 0.12706673, "policy/entropy_avg": 0.19923949, "val/ratio": 0.98832917, "val/ratio_var": 7.792e-05, "val/num_eos_tokens": 282, "lr": 8.87e-06, "episode": 896, "epoch": 0.04525253, "global_step/max_steps": "56/1238", "percentage": "4.52%", "elapsed_time": "38m 24s", "remaining_time": "13h 30m 33s"} +{"eps": 0, "objective/kl": 9.80964661, "objective/entropy": -50.02828217, "objective/non_score_reward": -0.4904823, "objective/rlhf_reward": 7.36889267, "objective/scores": 7.875, "policy/approxkl_avg": 0.00114043, "policy/clipfrac_avg": 0.00937813, "loss/policy_avg": 0.29207382, "loss/value_avg": 2.84465027, "val/clipfrac_avg": 0.22250551, "policy/entropy_avg": 0.32576752, "val/ratio": 0.98562229, "val/ratio_var": 4.662e-05, "val/num_eos_tokens": 524, "lr": 9.03e-06, "episode": 912, "epoch": 0.04606061, "global_step/max_steps": "57/1238", "percentage": "4.60%", "elapsed_time": "38m 57s", "remaining_time": "13h 27m 9s"} +{"eps": 0, "objective/kl": 8.7770052, "objective/entropy": -113.26528931, "objective/non_score_reward": -0.43885028, "objective/rlhf_reward": 9.4127121, "objective/scores": 9.875, "policy/approxkl_avg": 0.00083793, "policy/clipfrac_avg": 0.01237845, "loss/policy_avg": -0.40458733, "loss/value_avg": 2.15125394, "val/clipfrac_avg": 0.29461992, "policy/entropy_avg": 0.1743412, "val/ratio": 0.98873955, "val/ratio_var": 8.456e-05, "val/num_eos_tokens": 164, "lr": 9.19e-06, "episode": 928, "epoch": 0.04686869, "global_step/max_steps": "58/1238", "percentage": "4.68%", "elapsed_time": "39m 38s", "remaining_time": "13h 26m 24s"} +{"eps": 0, "objective/kl": 8.04400063, "objective/entropy": -101.63149261, "objective/non_score_reward": -0.40220004, "objective/rlhf_reward": 8.1681118, "objective/scores": 8.5625, "policy/approxkl_avg": 0.00914234, "policy/clipfrac_avg": 0.02357684, "loss/policy_avg": 0.15078622, "loss/value_avg": 1.80146074, "val/clipfrac_avg": 0.24946882, "policy/entropy_avg": 0.19042015, "val/ratio": 0.98881984, "val/ratio_var": 4.306e-05, "val/num_eos_tokens": 741, "lr": 9.35e-06, "episode": 944, "epoch": 0.04767677, "global_step/max_steps": "59/1238", "percentage": "4.77%", "elapsed_time": "40m 20s", "remaining_time": "13h 26m 12s"} +{"eps": 0, "objective/kl": 10.10027885, "objective/entropy": -105.13027954, "objective/non_score_reward": -0.50501394, "objective/rlhf_reward": 8.14732933, "objective/scores": 8.625, "policy/approxkl_avg": 0.0012912, "policy/clipfrac_avg": 0.01893148, "loss/policy_avg": 0.05019531, "loss/value_avg": 2.01290083, "val/clipfrac_avg": 0.12193798, "policy/entropy_avg": 0.21358871, "val/ratio": 0.98801488, "val/ratio_var": 6.14e-05, "val/num_eos_tokens": 80, "lr": 9.52e-06, "episode": 960, "epoch": 0.04848485, "global_step/max_steps": "60/1238", "percentage": "4.85%", "elapsed_time": "40m 58s", "remaining_time": "13h 24m 34s"} +{"eps": 0, "objective/kl": 7.40752792, "objective/entropy": -46.50836945, "objective/non_score_reward": -0.37037638, "objective/rlhf_reward": 9.98118591, "objective/scores": 10.375, "policy/approxkl_avg": 0.00103371, "policy/clipfrac_avg": 0.01406179, "loss/policy_avg": -0.14436418, "loss/value_avg": 2.70132971, "val/clipfrac_avg": 0.17322519, "policy/entropy_avg": 0.18859482, "val/ratio": 0.98792022, "val/ratio_var": 5.444e-05, "val/num_eos_tokens": 9, "lr": 9.68e-06, "episode": 976, "epoch": 0.04929293, "global_step/max_steps": "61/1238", "percentage": "4.93%", "elapsed_time": "41m 34s", "remaining_time": "13h 22m 12s"} +{"eps": 0, "objective/kl": 6.45943832, "objective/entropy": -104.46057129, "objective/non_score_reward": -0.32297194, "objective/rlhf_reward": 10.25124741, "objective/scores": 10.5625, "policy/approxkl_avg": 0.00089762, "policy/clipfrac_avg": 0.00845532, "loss/policy_avg": -0.43791005, "loss/value_avg": 2.99778342, "val/clipfrac_avg": 0.25087857, "policy/entropy_avg": 0.18575621, "val/ratio": 0.98832893, "val/ratio_var": 0.00015057, "val/num_eos_tokens": 47, "lr": 9.84e-06, "episode": 992, "epoch": 0.05010101, "global_step/max_steps": "62/1238", "percentage": "5.01%", "elapsed_time": "42m 12s", "remaining_time": "13h 20m 29s"} +{"eps": 0, "objective/kl": 8.99723816, "objective/entropy": -74.31800842, "objective/non_score_reward": -0.44986194, "objective/rlhf_reward": 8.12435627, "objective/scores": 8.5625, "policy/approxkl_avg": 0.00095336, "policy/clipfrac_avg": 0.01315314, "loss/policy_avg": 0.01164585, "loss/value_avg": 0.65884447, "val/clipfrac_avg": 0.19718438, "policy/entropy_avg": 0.20091534, "val/ratio": 0.98737979, "val/ratio_var": 5.344e-05, "val/num_eos_tokens": 181, "lr": 1e-05, "episode": 1008, "epoch": 0.05090909, "global_step/max_steps": "63/1238", "percentage": "5.09%", "elapsed_time": "42m 49s", "remaining_time": "13h 18m 35s"} +{"eps": 0, "objective/kl": 9.87634182, "objective/entropy": -115.92865753, "objective/non_score_reward": -0.49381709, "objective/rlhf_reward": 8.55305862, "objective/scores": 9.0625, "policy/approxkl_avg": 0.00093875, "policy/clipfrac_avg": 0.01131337, "loss/policy_avg": -0.17724252, "loss/value_avg": 2.51004052, "val/clipfrac_avg": 0.19327465, "policy/entropy_avg": 0.21930075, "val/ratio": 0.98809373, "val/ratio_var": 6.919e-05, "val/num_eos_tokens": 433, "lr": 1e-05, "episode": 1024, "epoch": 0.05171717, "global_step/max_steps": "64/1238", "percentage": "5.17%", "elapsed_time": "43m 50s", "remaining_time": "13h 24m 21s"} +{"eps": 0, "objective/kl": 12.10589027, "objective/entropy": -151.23590088, "objective/non_score_reward": -0.60529453, "objective/rlhf_reward": 7.0509553, "objective/scores": 7.65625, "policy/approxkl_avg": 0.00083117, "policy/clipfrac_avg": 0.02152154, "loss/policy_avg": -0.01736601, "loss/value_avg": 1.1105473, "val/clipfrac_avg": 0.2209429, "policy/entropy_avg": 0.1683712, "val/ratio": 0.98899168, "val/ratio_var": 6.484e-05, "val/num_eos_tokens": 350, "lr": 1e-05, "episode": 1040, "epoch": 0.05252525, "global_step/max_steps": "65/1238", "percentage": "5.25%", "elapsed_time": "44m 53s", "remaining_time": "13h 30m 8s"} +{"eps": 0, "objective/kl": 9.62144375, "objective/entropy": -121.24343872, "objective/non_score_reward": -0.48107219, "objective/rlhf_reward": 7.54041243, "objective/scores": 8.0, "policy/approxkl_avg": 0.00090604, "policy/clipfrac_avg": 0.01046483, "loss/policy_avg": -0.16295819, "loss/value_avg": 1.76386464, "val/clipfrac_avg": 0.20382188, "policy/entropy_avg": 0.17805845, "val/ratio": 0.98721623, "val/ratio_var": 0.00010222, "val/num_eos_tokens": 91, "lr": 1e-05, "episode": 1056, "epoch": 0.05333333, "global_step/max_steps": "66/1238", "percentage": "5.33%", "elapsed_time": "45m 35s", "remaining_time": "13h 29m 38s"} +{"eps": 0, "objective/kl": 9.10360909, "objective/entropy": -104.0590744, "objective/non_score_reward": -0.45518047, "objective/rlhf_reward": 9.69325638, "objective/scores": 10.125, "policy/approxkl_avg": 0.00112318, "policy/clipfrac_avg": 0.01493079, "loss/policy_avg": 0.15740007, "loss/value_avg": 3.37657356, "val/clipfrac_avg": 0.19888151, "policy/entropy_avg": 0.2234726, "val/ratio": 0.9885534, "val/ratio_var": 0.00011294, "val/num_eos_tokens": 446, "lr": 1e-05, "episode": 1072, "epoch": 0.05414141, "global_step/max_steps": "67/1238", "percentage": "5.41%", "elapsed_time": "46m 18s", "remaining_time": "13h 29m 20s"} +{"eps": 0, "objective/kl": 11.13963127, "objective/entropy": -45.16947556, "objective/non_score_reward": -0.55698156, "objective/rlhf_reward": 7.77895594, "objective/scores": 8.3125, "policy/approxkl_avg": 0.00120546, "policy/clipfrac_avg": 0.00824921, "loss/policy_avg": -0.08715461, "loss/value_avg": 0.94041902, "val/clipfrac_avg": 0.24800171, "policy/entropy_avg": 0.19194627, "val/ratio": 0.98705679, "val/ratio_var": 9.064e-05, "val/num_eos_tokens": 79, "lr": 1e-05, "episode": 1088, "epoch": 0.05494949, "global_step/max_steps": "68/1238", "percentage": "5.49%", "elapsed_time": "47m 7s", "remaining_time": "13h 30m 48s"} +{"eps": 0, "objective/kl": 8.01171589, "objective/entropy": -115.63008118, "objective/non_score_reward": -0.4005858, "objective/rlhf_reward": 10.53691483, "objective/scores": 10.9375, "policy/approxkl_avg": 0.00081256, "policy/clipfrac_avg": 0.01130526, "loss/policy_avg": -0.24446356, "loss/value_avg": 2.7453208, "val/clipfrac_avg": 0.2002556, "policy/entropy_avg": 0.14436889, "val/ratio": 0.99312377, "val/ratio_var": 0.00014785, "val/num_eos_tokens": 396, "lr": 1e-05, "episode": 1104, "epoch": 0.05575758, "global_step/max_steps": "69/1238", "percentage": "5.57%", "elapsed_time": "47m 35s", "remaining_time": "13h 26m 16s"} +{"eps": 0, "objective/kl": 10.50149727, "objective/entropy": -155.49316406, "objective/non_score_reward": -0.52507496, "objective/rlhf_reward": 9.31476879, "objective/scores": 9.8125, "policy/approxkl_avg": 0.00094765, "policy/clipfrac_avg": 0.00861178, "loss/policy_avg": -0.27069107, "loss/value_avg": 3.39486241, "val/clipfrac_avg": 0.25846598, "policy/entropy_avg": 0.17398953, "val/ratio": 0.98791748, "val/ratio_var": 0.00018113, "val/num_eos_tokens": 621, "lr": 1e-05, "episode": 1120, "epoch": 0.05656566, "global_step/max_steps": "70/1238", "percentage": "5.65%", "elapsed_time": "48m 18s", "remaining_time": "13h 26m 1s"} +{"eps": 0, "objective/kl": 11.062006, "objective/entropy": -32.39612579, "objective/non_score_reward": -0.55310035, "objective/rlhf_reward": 10.07189941, "objective/scores": 10.625, "policy/approxkl_avg": 0.00106518, "policy/clipfrac_avg": 0.01812572, "loss/policy_avg": 0.06734678, "loss/value_avg": 2.07879591, "val/clipfrac_avg": 0.22745997, "policy/entropy_avg": 0.20269871, "val/ratio": 0.98762727, "val/ratio_var": 4.869e-05, "val/num_eos_tokens": 56, "lr": 1e-05, "episode": 1136, "epoch": 0.05737374, "global_step/max_steps": "71/1238", "percentage": "5.74%", "elapsed_time": "48m 58s", "remaining_time": "13h 25m 0s"} +{"eps": 0, "objective/kl": 8.33562851, "objective/entropy": -84.5282135, "objective/non_score_reward": -0.41678143, "objective/rlhf_reward": 7.70821857, "objective/scores": 8.125, "policy/approxkl_avg": 0.0018769, "policy/clipfrac_avg": 0.03232686, "loss/policy_avg": 0.36056173, "loss/value_avg": 3.14741135, "val/clipfrac_avg": 0.21321574, "policy/entropy_avg": 0.30700183, "val/ratio": 0.99178648, "val/ratio_var": 8.937e-05, "val/num_eos_tokens": 447, "lr": 1e-05, "episode": 1152, "epoch": 0.05818182, "global_step/max_steps": "72/1238", "percentage": "5.82%", "elapsed_time": "49m 27s", "remaining_time": "13h 20m 56s"} +{"eps": 0, "objective/kl": 14.06869221, "objective/entropy": -81.50756836, "objective/non_score_reward": -0.70343465, "objective/rlhf_reward": 8.04656601, "objective/scores": 8.75, "policy/approxkl_avg": 0.00293864, "policy/clipfrac_avg": 0.04300766, "loss/policy_avg": -0.04088821, "loss/value_avg": 1.45049429, "val/clipfrac_avg": 0.0704298, "policy/entropy_avg": 0.19576502, "val/ratio": 0.98799801, "val/ratio_var": 7.575e-05, "val/num_eos_tokens": 149, "lr": 1e-05, "episode": 1168, "epoch": 0.0589899, "global_step/max_steps": "73/1238", "percentage": "5.90%", "elapsed_time": "50m 19s", "remaining_time": "13h 23m 9s"} +{"eps": 0, "objective/kl": 9.6719923, "objective/entropy": -95.04641724, "objective/non_score_reward": -0.48359963, "objective/rlhf_reward": 10.77421284, "objective/scores": 11.25, "policy/approxkl_avg": 0.00076051, "policy/clipfrac_avg": 0.00979708, "loss/policy_avg": -0.14120109, "loss/value_avg": 2.75473452, "val/clipfrac_avg": 0.15541101, "policy/entropy_avg": 0.14811003, "val/ratio": 0.990731, "val/ratio_var": 6.645e-05, "val/num_eos_tokens": 261, "lr": 1e-05, "episode": 1184, "epoch": 0.05979798, "global_step/max_steps": "74/1238", "percentage": "5.98%", "elapsed_time": "50m 43s", "remaining_time": "13h 17m 54s"} +{"eps": 0, "objective/kl": 10.37656784, "objective/entropy": -133.35256958, "objective/non_score_reward": -0.51882839, "objective/rlhf_reward": 5.96749973, "objective/scores": 6.5, "policy/approxkl_avg": 0.00104538, "policy/clipfrac_avg": 0.02754609, "loss/policy_avg": 0.45924872, "loss/value_avg": 2.21863151, "val/clipfrac_avg": 0.25358582, "policy/entropy_avg": 0.23077679, "val/ratio": 0.99033749, "val/ratio_var": 3.861e-05, "val/num_eos_tokens": 319, "lr": 1e-05, "episode": 1200, "epoch": 0.06060606, "global_step/max_steps": "75/1238", "percentage": "6.06%", "elapsed_time": "51m 23s", "remaining_time": "13h 16m 59s"} +{"eps": 0, "objective/kl": 11.16040039, "objective/entropy": -95.95547485, "objective/non_score_reward": -0.55802, "objective/rlhf_reward": 9.32869911, "objective/scores": 9.875, "policy/approxkl_avg": 0.00085199, "policy/clipfrac_avg": 0.0141358, "loss/policy_avg": 0.05463395, "loss/value_avg": 1.46578777, "val/clipfrac_avg": 0.11248328, "policy/entropy_avg": 0.16746521, "val/ratio": 0.99046403, "val/ratio_var": 5.162e-05, "val/num_eos_tokens": 231, "lr": 1e-05, "episode": 1216, "epoch": 0.06141414, "global_step/max_steps": "76/1238", "percentage": "6.14%", "elapsed_time": "52m 1s", "remaining_time": "13h 15m 31s"} +{"eps": 0, "objective/kl": 8.94723034, "objective/entropy": -142.79956055, "objective/non_score_reward": -0.44736153, "objective/rlhf_reward": 8.0331068, "objective/scores": 8.5, "policy/approxkl_avg": 0.00073969, "policy/clipfrac_avg": 0.00677315, "loss/policy_avg": 0.06325521, "loss/value_avg": 1.28276479, "val/clipfrac_avg": 0.07109328, "policy/entropy_avg": 0.16584384, "val/ratio": 0.99117881, "val/ratio_var": 5.505e-05, "val/num_eos_tokens": 303, "lr": 1e-05, "episode": 1232, "epoch": 0.06222222, "global_step/max_steps": "77/1238", "percentage": "6.22%", "elapsed_time": "52m 51s", "remaining_time": "13h 16m 55s"} +{"eps": 0, "objective/kl": 9.97001076, "objective/entropy": -138.09356689, "objective/non_score_reward": -0.49850059, "objective/rlhf_reward": 10.21243668, "objective/scores": 10.6875, "policy/approxkl_avg": 0.00090716, "policy/clipfrac_avg": 0.00806038, "loss/policy_avg": -0.76631916, "loss/value_avg": 2.67302561, "val/clipfrac_avg": 0.25913984, "policy/entropy_avg": 0.15853584, "val/ratio": 0.99083424, "val/ratio_var": 5.45e-05, "val/num_eos_tokens": 47, "lr": 1e-05, "episode": 1248, "epoch": 0.0630303, "global_step/max_steps": "78/1238", "percentage": "6.30%", "elapsed_time": "53m 29s", "remaining_time": "13h 15m 35s"} +{"eps": 0, "objective/kl": 8.2618742, "objective/entropy": -55.90776062, "objective/non_score_reward": -0.41309375, "objective/rlhf_reward": 8.18065643, "objective/scores": 8.625, "policy/approxkl_avg": 0.00221771, "policy/clipfrac_avg": 0.01831599, "loss/policy_avg": 0.30310458, "loss/value_avg": 2.48160934, "val/clipfrac_avg": 0.29067862, "policy/entropy_avg": 0.19953299, "val/ratio": 0.98706293, "val/ratio_var": 7.513e-05, "val/num_eos_tokens": 16, "lr": 1e-05, "episode": 1264, "epoch": 0.06383838, "global_step/max_steps": "79/1238", "percentage": "6.38%", "elapsed_time": "54m 5s", "remaining_time": "13h 13m 30s"} +{"eps": 0, "objective/kl": 10.4533062, "objective/entropy": -114.97703552, "objective/non_score_reward": -0.52266532, "objective/rlhf_reward": 8.91092873, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00082728, "policy/clipfrac_avg": 0.0084436, "loss/policy_avg": -0.13517441, "loss/value_avg": 1.40773249, "val/clipfrac_avg": 0.24559587, "policy/entropy_avg": 0.16773379, "val/ratio": 0.9903726, "val/ratio_var": 9.728e-05, "val/num_eos_tokens": 597, "lr": 9.99e-06, "episode": 1280, "epoch": 0.06464646, "global_step/max_steps": "80/1238", "percentage": "6.46%", "elapsed_time": "54m 51s", "remaining_time": "13h 14m 4s"} +{"eps": 0, "objective/kl": 8.21436501, "objective/entropy": -65.29562378, "objective/non_score_reward": -0.41071823, "objective/rlhf_reward": 9.48771858, "objective/scores": 9.875, "policy/approxkl_avg": 0.0007956, "policy/clipfrac_avg": 0.0067076, "loss/policy_avg": -0.22855638, "loss/value_avg": 2.56282783, "val/clipfrac_avg": 0.1218106, "policy/entropy_avg": 0.13605309, "val/ratio": 0.9903298, "val/ratio_var": 5.526e-05, "val/num_eos_tokens": 276, "lr": 9.99e-06, "episode": 1296, "epoch": 0.06545455, "global_step/max_steps": "81/1238", "percentage": "6.54%", "elapsed_time": "55m 27s", "remaining_time": "13h 12m 6s"} +{"eps": 0, "objective/kl": 8.70901775, "objective/entropy": -71.39965057, "objective/non_score_reward": -0.43545088, "objective/rlhf_reward": 7.35947084, "objective/scores": 7.78125, "policy/approxkl_avg": 0.00095097, "policy/clipfrac_avg": 0.01494086, "loss/policy_avg": 0.37310424, "loss/value_avg": 1.8982327, "val/clipfrac_avg": 0.34438807, "policy/entropy_avg": 0.18790436, "val/ratio": 0.98744512, "val/ratio_var": 3.962e-05, "val/num_eos_tokens": 276, "lr": 9.99e-06, "episode": 1312, "epoch": 0.06626263, "global_step/max_steps": "82/1238", "percentage": "6.62%", "elapsed_time": "56m 4s", "remaining_time": "13h 10m 30s"} +{"eps": 0, "objective/kl": 9.87769222, "objective/entropy": -186.06890869, "objective/non_score_reward": -0.49388456, "objective/rlhf_reward": 7.85377169, "objective/scores": 8.375, "policy/approxkl_avg": 0.0008738, "policy/clipfrac_avg": 0.03074601, "loss/policy_avg": -0.06645296, "loss/value_avg": 1.95434904, "val/clipfrac_avg": 0.21427029, "policy/entropy_avg": 0.27783632, "val/ratio": 0.98956919, "val/ratio_var": 0.0001006, "val/num_eos_tokens": 434, "lr": 9.99e-06, "episode": 1328, "epoch": 0.06707071, "global_step/max_steps": "83/1238", "percentage": "6.70%", "elapsed_time": "56m 48s", "remaining_time": "13h 10m 36s"} +{"eps": 0, "objective/kl": 10.45096207, "objective/entropy": -112.02441406, "objective/non_score_reward": -0.52254808, "objective/rlhf_reward": 8.91495132, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00106057, "policy/clipfrac_avg": 0.01699572, "loss/policy_avg": 0.0639994, "loss/value_avg": 1.29278409, "val/clipfrac_avg": 0.0605298, "policy/entropy_avg": 0.15537548, "val/ratio": 0.98914897, "val/ratio_var": 7.554e-05, "val/num_eos_tokens": 693, "lr": 9.99e-06, "episode": 1344, "epoch": 0.06787879, "global_step/max_steps": "84/1238", "percentage": "6.79%", "elapsed_time": "57m 25s", "remaining_time": "13h 9m 1s"} +{"eps": 0, "objective/kl": 7.84175062, "objective/entropy": -110.17214966, "objective/non_score_reward": -0.39208752, "objective/rlhf_reward": 9.15478706, "objective/scores": 9.5625, "policy/approxkl_avg": 0.00115109, "policy/clipfrac_avg": 0.02971536, "loss/policy_avg": 0.05262536, "loss/value_avg": 1.90761304, "val/clipfrac_avg": 0.13365106, "policy/entropy_avg": 0.14819884, "val/ratio": 0.99085265, "val/ratio_var": 6.176e-05, "val/num_eos_tokens": 172, "lr": 9.99e-06, "episode": 1360, "epoch": 0.06868687, "global_step/max_steps": "85/1238", "percentage": "6.87%", "elapsed_time": "57m 56s", "remaining_time": "13h 6m 3s"} +{"eps": 0, "objective/kl": 10.16928959, "objective/entropy": -125.11438751, "objective/non_score_reward": -0.50846446, "objective/rlhf_reward": 7.85872269, "objective/scores": 8.375, "policy/approxkl_avg": 0.00089277, "policy/clipfrac_avg": 0.01469754, "loss/policy_avg": 0.29705334, "loss/value_avg": 2.05938053, "val/clipfrac_avg": 0.16572037, "policy/entropy_avg": 0.17176449, "val/ratio": 0.98885876, "val/ratio_var": 8.455e-05, "val/num_eos_tokens": 262, "lr": 9.99e-06, "episode": 1376, "epoch": 0.06949495, "global_step/max_steps": "86/1238", "percentage": "6.95%", "elapsed_time": "58m 34s", "remaining_time": "13h 4m 40s"} +{"eps": 0, "objective/kl": 9.3726263, "objective/entropy": -110.98075867, "objective/non_score_reward": -0.46863133, "objective/rlhf_reward": 9.06261921, "objective/scores": 9.5, "policy/approxkl_avg": 0.00092937, "policy/clipfrac_avg": 0.01337194, "loss/policy_avg": -0.31014776, "loss/value_avg": 1.60577989, "val/clipfrac_avg": 0.141675, "policy/entropy_avg": 0.20984077, "val/ratio": 0.98939556, "val/ratio_var": 0.00011555, "val/num_eos_tokens": 219, "lr": 9.99e-06, "episode": 1392, "epoch": 0.07030303, "global_step/max_steps": "87/1238", "percentage": "7.03%", "elapsed_time": "59m 13s", "remaining_time": "13h 3m 38s"} +{"eps": 0, "objective/kl": 8.59924889, "objective/entropy": -119.25131989, "objective/non_score_reward": -0.42996243, "objective/rlhf_reward": 9.56222534, "objective/scores": 10.0, "policy/approxkl_avg": 0.00086327, "policy/clipfrac_avg": 0.01425031, "loss/policy_avg": -0.36583343, "loss/value_avg": 1.49733841, "val/clipfrac_avg": 0.19998914, "policy/entropy_avg": 0.1634059, "val/ratio": 0.99014723, "val/ratio_var": 9.214e-05, "val/num_eos_tokens": 380, "lr": 9.99e-06, "episode": 1408, "epoch": 0.07111111, "global_step/max_steps": "88/1238", "percentage": "7.11%", "elapsed_time": "59m 50s", "remaining_time": "13h 1m 59s"} +{"eps": 0, "objective/kl": 7.49328566, "objective/entropy": -137.16841125, "objective/non_score_reward": -0.37466431, "objective/rlhf_reward": 6.91439819, "objective/scores": 7.28125, "policy/approxkl_avg": 0.00100051, "policy/clipfrac_avg": 0.01220254, "loss/policy_avg": 0.39670557, "loss/value_avg": 2.17699194, "val/clipfrac_avg": 0.22100285, "policy/entropy_avg": 0.13602829, "val/ratio": 0.99342227, "val/ratio_var": 0.00017903, "val/num_eos_tokens": 391, "lr": 9.99e-06, "episode": 1424, "epoch": 0.07191919, "global_step/max_steps": "89/1238", "percentage": "7.19%", "elapsed_time": "1h 0m 27s", "remaining_time": "13h 0m 34s"} +{"eps": 0, "objective/kl": 9.35251236, "objective/entropy": -102.01996613, "objective/non_score_reward": -0.46762562, "objective/rlhf_reward": 8.86831188, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0007847, "policy/clipfrac_avg": 0.00758838, "loss/policy_avg": -0.16625538, "loss/value_avg": 0.6386686, "val/clipfrac_avg": 0.26963502, "policy/entropy_avg": 0.18400812, "val/ratio": 0.99047554, "val/ratio_var": 4.935e-05, "val/num_eos_tokens": 381, "lr": 9.99e-06, "episode": 1440, "epoch": 0.07272727, "global_step/max_steps": "90/1238", "percentage": "7.27%", "elapsed_time": "1h 0m 58s", "remaining_time": "12h 57m 45s"} +{"eps": 0, "objective/kl": 6.70796442, "objective/entropy": -52.68136215, "objective/non_score_reward": -0.33539826, "objective/rlhf_reward": 10.49272633, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00121863, "policy/clipfrac_avg": 0.00611027, "loss/policy_avg": 0.06461674, "loss/value_avg": 1.8350507, "val/clipfrac_avg": 0.15903372, "policy/entropy_avg": 0.21361065, "val/ratio": 0.98496145, "val/ratio_var": 0.00012401, "val/num_eos_tokens": 142, "lr": 9.99e-06, "episode": 1456, "epoch": 0.07353535, "global_step/max_steps": "91/1238", "percentage": "7.35%", "elapsed_time": "1h 1m 18s", "remaining_time": "12h 52m 50s"} +{"eps": 0, "objective/kl": 11.08205223, "objective/entropy": -110.0786438, "objective/non_score_reward": -0.55410266, "objective/rlhf_reward": 8.5708971, "objective/scores": 9.125, "policy/approxkl_avg": 0.00086472, "policy/clipfrac_avg": 0.01752229, "loss/policy_avg": 0.28746617, "loss/value_avg": 1.23556757, "val/clipfrac_avg": 0.11287079, "policy/entropy_avg": 0.1840806, "val/ratio": 0.98822284, "val/ratio_var": 6.304e-05, "val/num_eos_tokens": 48, "lr": 9.99e-06, "episode": 1472, "epoch": 0.07434343, "global_step/max_steps": "92/1238", "percentage": "7.43%", "elapsed_time": "1h 1m 58s", "remaining_time": "12h 52m 3s"} +{"eps": 0, "objective/kl": 10.31765556, "objective/entropy": -110.14735413, "objective/non_score_reward": -0.51588279, "objective/rlhf_reward": 10.44505501, "objective/scores": 10.9375, "policy/approxkl_avg": 0.00079859, "policy/clipfrac_avg": 0.00367145, "loss/policy_avg": -1.08345532, "loss/value_avg": 2.37427497, "val/clipfrac_avg": 0.26723406, "policy/entropy_avg": 0.15915787, "val/ratio": 0.98924959, "val/ratio_var": 6.818e-05, "val/num_eos_tokens": 348, "lr": 9.98e-06, "episode": 1488, "epoch": 0.07515152, "global_step/max_steps": "93/1238", "percentage": "7.51%", "elapsed_time": "1h 2m 33s", "remaining_time": "12h 50m 17s"} +{"eps": 0, "objective/kl": 11.67657852, "objective/entropy": -26.38776779, "objective/non_score_reward": -0.58382893, "objective/rlhf_reward": 8.23257637, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00300202, "policy/clipfrac_avg": 0.01405197, "loss/policy_avg": -0.01678584, "loss/value_avg": 2.52266788, "val/clipfrac_avg": 0.19883442, "policy/entropy_avg": 0.23279858, "val/ratio": 0.98663819, "val/ratio_var": 6.661e-05, "val/num_eos_tokens": 74, "lr": 9.98e-06, "episode": 1504, "epoch": 0.0759596, "global_step/max_steps": "94/1238", "percentage": "7.59%", "elapsed_time": "1h 3m 5s", "remaining_time": "12h 47m 49s"} +{"eps": 0, "objective/kl": 11.78497601, "objective/entropy": -111.26895142, "objective/non_score_reward": -0.58924878, "objective/rlhf_reward": 9.95762634, "objective/scores": 10.5625, "policy/approxkl_avg": 0.00325959, "policy/clipfrac_avg": 0.0253926, "loss/policy_avg": -0.13661003, "loss/value_avg": 2.23880076, "val/clipfrac_avg": 0.15024024, "policy/entropy_avg": 0.23312855, "val/ratio": 0.9824518, "val/ratio_var": 0.00033877, "val/num_eos_tokens": 15, "lr": 9.98e-06, "episode": 1520, "epoch": 0.07676768, "global_step/max_steps": "95/1238", "percentage": "7.67%", "elapsed_time": "1h 3m 59s", "remaining_time": "12h 49m 59s"} +{"eps": 0, "objective/kl": 9.11925125, "objective/entropy": -67.97806549, "objective/non_score_reward": -0.45596254, "objective/rlhf_reward": 7.07138109, "objective/scores": 7.53125, "policy/approxkl_avg": 0.00093705, "policy/clipfrac_avg": 0.02322861, "loss/policy_avg": 0.31284308, "loss/value_avg": 3.77775359, "val/clipfrac_avg": 0.30271417, "policy/entropy_avg": 0.19547653, "val/ratio": 0.98777866, "val/ratio_var": 7.666e-05, "val/num_eos_tokens": 180, "lr": 9.98e-06, "episode": 1536, "epoch": 0.07757576, "global_step/max_steps": "96/1238", "percentage": "7.75%", "elapsed_time": "1h 4m 28s", "remaining_time": "12h 46m 56s"} +{"eps": 0, "objective/kl": 10.92871666, "objective/entropy": -168.77575684, "objective/non_score_reward": -0.54643577, "objective/rlhf_reward": 7.79340792, "objective/scores": 8.3125, "policy/approxkl_avg": 0.00108027, "policy/clipfrac_avg": 0.01312133, "loss/policy_avg": -0.26056254, "loss/value_avg": 1.99768376, "val/clipfrac_avg": 0.17170179, "policy/entropy_avg": 0.14757729, "val/ratio": 0.99210817, "val/ratio_var": 0.00020362, "val/num_eos_tokens": 530, "lr": 9.98e-06, "episode": 1552, "epoch": 0.07838384, "global_step/max_steps": "97/1238", "percentage": "7.84%", "elapsed_time": "1h 5m 24s", "remaining_time": "12h 49m 22s"} +{"eps": 0, "objective/kl": 8.58163643, "objective/entropy": -88.97151184, "objective/non_score_reward": -0.42908183, "objective/rlhf_reward": 7.73888731, "objective/scores": 8.1875, "policy/approxkl_avg": 0.0008208, "policy/clipfrac_avg": 0.01489139, "loss/policy_avg": 0.20252731, "loss/value_avg": 1.88692772, "val/clipfrac_avg": 0.14818436, "policy/entropy_avg": 0.24853516, "val/ratio": 0.99012113, "val/ratio_var": 6.447e-05, "val/num_eos_tokens": 287, "lr": 9.98e-06, "episode": 1568, "epoch": 0.07919192, "global_step/max_steps": "98/1238", "percentage": "7.92%", "elapsed_time": "1h 5m 59s", "remaining_time": "12h 47m 42s"} +{"eps": 0, "objective/kl": 10.81656456, "objective/entropy": -70.99946594, "objective/non_score_reward": -0.54082829, "objective/rlhf_reward": 8.1232338, "objective/scores": 8.6875, "policy/approxkl_avg": 0.00238238, "policy/clipfrac_avg": 0.03098738, "loss/policy_avg": 0.4129256, "loss/value_avg": 1.67778707, "val/clipfrac_avg": 0.17884067, "policy/entropy_avg": 0.19532859, "val/ratio": 0.98638719, "val/ratio_var": 0.0001791, "val/num_eos_tokens": 287, "lr": 9.98e-06, "episode": 1584, "epoch": 0.08, "global_step/max_steps": "99/1238", "percentage": "8.00%", "elapsed_time": "1h 6m 32s", "remaining_time": "12h 45m 32s"} +{"eps": 0, "objective/kl": 10.95974827, "objective/entropy": -94.05448914, "objective/non_score_reward": -0.54798746, "objective/rlhf_reward": 9.29576206, "objective/scores": 9.875, "policy/approxkl_avg": 0.00103384, "policy/clipfrac_avg": 0.00906378, "loss/policy_avg": 0.07570861, "loss/value_avg": 0.57864887, "val/clipfrac_avg": 0.13884296, "policy/entropy_avg": 0.19421935, "val/ratio": 0.98734707, "val/ratio_var": 7.108e-05, "val/num_eos_tokens": 14, "lr": 9.98e-06, "episode": 1600, "epoch": 0.08080808, "global_step/max_steps": "100/1238", "percentage": "8.08%", "elapsed_time": "1h 7m 18s", "remaining_time": "12h 46m 1s"} +{"eps": 0, "objective/kl": 9.48241997, "objective/entropy": -147.22831726, "objective/non_score_reward": -0.47412103, "objective/rlhf_reward": 9.45361328, "objective/scores": 9.9375, "policy/approxkl_avg": 0.00086625, "policy/clipfrac_avg": 0.01102792, "loss/policy_avg": 0.18178648, "loss/value_avg": 3.24693656, "val/clipfrac_avg": 0.21203864, "policy/entropy_avg": 0.18662024, "val/ratio": 0.98940641, "val/ratio_var": 9.009e-05, "val/num_eos_tokens": 62, "lr": 9.97e-06, "episode": 1616, "epoch": 0.08161616, "global_step/max_steps": "101/1238", "percentage": "8.16%", "elapsed_time": "1h 8m 37s", "remaining_time": "12h 52m 35s"} +{"eps": 0, "objective/kl": 11.22801781, "objective/entropy": -102.03694916, "objective/non_score_reward": -0.56140089, "objective/rlhf_reward": 9.19445801, "objective/scores": 9.75, "policy/approxkl_avg": 0.00092448, "policy/clipfrac_avg": 0.02670031, "loss/policy_avg": -0.36225817, "loss/value_avg": 3.51721621, "val/clipfrac_avg": 0.22908944, "policy/entropy_avg": 0.19944334, "val/ratio": 0.98838305, "val/ratio_var": 8.266e-05, "val/num_eos_tokens": 360, "lr": 9.97e-06, "episode": 1632, "epoch": 0.08242424, "global_step/max_steps": "102/1238", "percentage": "8.24%", "elapsed_time": "1h 9m 14s", "remaining_time": "12h 51m 7s"} +{"eps": 0, "objective/kl": 11.77578068, "objective/entropy": -142.39923096, "objective/non_score_reward": -0.58878905, "objective/rlhf_reward": 9.02839851, "objective/scores": 9.625, "policy/approxkl_avg": 0.00097675, "policy/clipfrac_avg": 0.01352252, "loss/policy_avg": 0.00434394, "loss/value_avg": 3.05381966, "val/clipfrac_avg": 0.21960068, "policy/entropy_avg": 0.26236868, "val/ratio": 0.98831749, "val/ratio_var": 0.00010565, "val/num_eos_tokens": 9, "lr": 9.97e-06, "episode": 1648, "epoch": 0.08323232, "global_step/max_steps": "103/1238", "percentage": "8.32%", "elapsed_time": "1h 10m 16s", "remaining_time": "12h 54m 27s"} +{"eps": 0, "objective/kl": 12.53150177, "objective/entropy": -71.44817352, "objective/non_score_reward": -0.62657511, "objective/rlhf_reward": 8.52576828, "objective/scores": 9.125, "policy/approxkl_avg": 0.00105123, "policy/clipfrac_avg": 0.03077286, "loss/policy_avg": 0.49317449, "loss/value_avg": 2.04217625, "val/clipfrac_avg": 0.23763742, "policy/entropy_avg": 0.20035934, "val/ratio": 0.98624671, "val/ratio_var": 0.00010009, "val/num_eos_tokens": 23, "lr": 9.97e-06, "episode": 1664, "epoch": 0.0840404, "global_step/max_steps": "104/1238", "percentage": "8.40%", "elapsed_time": "1h 10m 52s", "remaining_time": "12h 52m 46s"} +{"eps": 0, "objective/kl": 11.11354065, "objective/entropy": -60.47146225, "objective/non_score_reward": -0.55567706, "objective/rlhf_reward": 8.44432259, "objective/scores": 9.0, "policy/approxkl_avg": 0.0010112, "policy/clipfrac_avg": 0.01647218, "loss/policy_avg": 0.15791772, "loss/value_avg": 1.99508762, "val/clipfrac_avg": 0.27714849, "policy/entropy_avg": 0.15747523, "val/ratio": 0.98978662, "val/ratio_var": 4.675e-05, "val/num_eos_tokens": 26, "lr": 9.97e-06, "episode": 1680, "epoch": 0.08484848, "global_step/max_steps": "105/1238", "percentage": "8.48%", "elapsed_time": "1h 11m 20s", "remaining_time": "12h 49m 47s"} +{"eps": 0, "objective/kl": 12.04114342, "objective/entropy": -109.22718811, "objective/non_score_reward": -0.60205722, "objective/rlhf_reward": 9.97606754, "objective/scores": 10.5625, "policy/approxkl_avg": 0.00196066, "policy/clipfrac_avg": 0.02854878, "loss/policy_avg": 0.14141756, "loss/value_avg": 1.61327529, "val/clipfrac_avg": 0.16923228, "policy/entropy_avg": 0.14309561, "val/ratio": 0.98959976, "val/ratio_var": 5.035e-05, "val/num_eos_tokens": 19, "lr": 9.97e-06, "episode": 1696, "epoch": 0.08565657, "global_step/max_steps": "106/1238", "percentage": "8.56%", "elapsed_time": "1h 12m 1s", "remaining_time": "12h 49m 6s"} +{"eps": 0, "objective/kl": 11.27841377, "objective/entropy": -166.47471619, "objective/non_score_reward": -0.56392074, "objective/rlhf_reward": 11.27201653, "objective/scores": 11.8125, "policy/approxkl_avg": 0.00081307, "policy/clipfrac_avg": 0.00993924, "loss/policy_avg": -0.13957123, "loss/value_avg": 0.89858395, "val/clipfrac_avg": 0.10860807, "policy/entropy_avg": 0.13879013, "val/ratio": 0.99106276, "val/ratio_var": 4.762e-05, "val/num_eos_tokens": 232, "lr": 9.97e-06, "episode": 1712, "epoch": 0.08646465, "global_step/max_steps": "107/1238", "percentage": "8.64%", "elapsed_time": "1h 12m 43s", "remaining_time": "12h 48m 43s"} +{"eps": 0, "objective/kl": 10.21388245, "objective/entropy": -81.45556641, "objective/non_score_reward": -0.51069415, "objective/rlhf_reward": 9.043993, "objective/scores": 9.5625, "policy/approxkl_avg": 0.00120083, "policy/clipfrac_avg": 0.01031427, "loss/policy_avg": -0.11392714, "loss/value_avg": 1.13671052, "val/clipfrac_avg": 0.16719487, "policy/entropy_avg": 0.22310293, "val/ratio": 0.98736429, "val/ratio_var": 0.00011893, "val/num_eos_tokens": 724, "lr": 9.96e-06, "episode": 1728, "epoch": 0.08727273, "global_step/max_steps": "108/1238", "percentage": "8.72%", "elapsed_time": "1h 13m 21s", "remaining_time": "12h 47m 34s"} +{"eps": 0, "objective/kl": 17.01079369, "objective/entropy": -23.41170502, "objective/non_score_reward": -0.85053968, "objective/rlhf_reward": 6.88774157, "objective/scores": 7.75, "policy/approxkl_avg": 0.00171468, "policy/clipfrac_avg": 0.00772217, "loss/policy_avg": 0.02373083, "loss/value_avg": 0.75513935, "val/clipfrac_avg": 0.26515225, "policy/entropy_avg": 0.24081039, "val/ratio": 0.98395318, "val/ratio_var": 0.00013079, "val/num_eos_tokens": 92, "lr": 9.96e-06, "episode": 1744, "epoch": 0.08808081, "global_step/max_steps": "109/1238", "percentage": "8.80%", "elapsed_time": "1h 13m 52s", "remaining_time": "12h 45m 12s"} +{"eps": 0, "objective/kl": 17.35074615, "objective/entropy": -49.22315216, "objective/non_score_reward": -0.86753726, "objective/rlhf_reward": 7.57386875, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00108193, "policy/clipfrac_avg": 0.01232925, "loss/policy_avg": -0.02932338, "loss/value_avg": 0.6305722, "val/clipfrac_avg": 0.14187291, "policy/entropy_avg": 0.1938591, "val/ratio": 0.98764616, "val/ratio_var": 5.469e-05, "val/num_eos_tokens": 469, "lr": 9.96e-06, "episode": 1760, "epoch": 0.08888889, "global_step/max_steps": "110/1238", "percentage": "8.89%", "elapsed_time": "1h 14m 28s", "remaining_time": "12h 43m 43s"} +{"eps": 0, "objective/kl": 13.64469051, "objective/entropy": -99.91004181, "objective/non_score_reward": -0.68223453, "objective/rlhf_reward": 9.59120274, "objective/scores": 10.25, "policy/approxkl_avg": 0.00132188, "policy/clipfrac_avg": 0.02359712, "loss/policy_avg": -0.09420577, "loss/value_avg": 1.10376406, "val/clipfrac_avg": 0.16007894, "policy/entropy_avg": 0.19385338, "val/ratio": 0.98767525, "val/ratio_var": 9.704e-05, "val/num_eos_tokens": 23, "lr": 9.96e-06, "episode": 1776, "epoch": 0.08969697, "global_step/max_steps": "111/1238", "percentage": "8.97%", "elapsed_time": "1h 15m 6s", "remaining_time": "12h 42m 37s"} +{"eps": 0, "objective/kl": 15.63341808, "objective/entropy": -43.24087524, "objective/non_score_reward": -0.78167093, "objective/rlhf_reward": 7.91364098, "objective/scores": 8.6875, "policy/approxkl_avg": 0.00288602, "policy/clipfrac_avg": 0.01270105, "loss/policy_avg": -0.0123039, "loss/value_avg": 1.27620864, "val/clipfrac_avg": 0.23217325, "policy/entropy_avg": 0.23091507, "val/ratio": 0.98615873, "val/ratio_var": 0.00037474, "val/num_eos_tokens": 18, "lr": 9.96e-06, "episode": 1792, "epoch": 0.09050505, "global_step/max_steps": "112/1238", "percentage": "9.05%", "elapsed_time": "1h 15m 45s", "remaining_time": "12h 41m 36s"} +{"eps": 0, "objective/kl": 12.85316086, "objective/entropy": -69.88031769, "objective/non_score_reward": -0.64265805, "objective/rlhf_reward": 9.56827927, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00113245, "policy/clipfrac_avg": 0.00449784, "loss/policy_avg": -0.5104568, "loss/value_avg": 2.16391635, "val/clipfrac_avg": 0.28476185, "policy/entropy_avg": 0.16321039, "val/ratio": 0.98939282, "val/ratio_var": 6.093e-05, "val/num_eos_tokens": 59, "lr": 9.96e-06, "episode": 1808, "epoch": 0.09131313, "global_step/max_steps": "113/1238", "percentage": "9.13%", "elapsed_time": "1h 16m 27s", "remaining_time": "12h 41m 14s"} +{"eps": 0, "objective/kl": 13.04114437, "objective/entropy": -92.63809204, "objective/non_score_reward": -0.65205717, "objective/rlhf_reward": 10.61356735, "objective/scores": 11.25, "policy/approxkl_avg": 0.00099896, "policy/clipfrac_avg": 0.01249834, "loss/policy_avg": -0.57007217, "loss/value_avg": 3.39098167, "val/clipfrac_avg": 0.16689906, "policy/entropy_avg": 0.15574789, "val/ratio": 0.98842347, "val/ratio_var": 0.00011906, "val/num_eos_tokens": 337, "lr": 9.95e-06, "episode": 1824, "epoch": 0.09212121, "global_step/max_steps": "114/1238", "percentage": "9.21%", "elapsed_time": "1h 16m 54s", "remaining_time": "12h 38m 17s"} +{"eps": 0, "objective/kl": 11.18808746, "objective/entropy": -62.34325027, "objective/non_score_reward": -0.55940437, "objective/rlhf_reward": 9.19840813, "objective/scores": 9.75, "policy/approxkl_avg": 0.00080789, "policy/clipfrac_avg": 0.00809546, "loss/policy_avg": -0.03700163, "loss/value_avg": 1.2843833, "val/clipfrac_avg": 0.14679772, "policy/entropy_avg": 0.16070127, "val/ratio": 0.99077994, "val/ratio_var": 6.536e-05, "val/num_eos_tokens": 491, "lr": 9.95e-06, "episode": 1840, "epoch": 0.09292929, "global_step/max_steps": "115/1238", "percentage": "9.29%", "elapsed_time": "1h 17m 28s", "remaining_time": "12h 36m 31s"} +{"eps": 0, "objective/kl": 13.15189552, "objective/entropy": -67.57776642, "objective/non_score_reward": -0.6575948, "objective/rlhf_reward": 7.94396782, "objective/scores": 8.625, "policy/approxkl_avg": 0.00152462, "policy/clipfrac_avg": 0.03400165, "loss/policy_avg": 0.39205495, "loss/value_avg": 4.03825665, "val/clipfrac_avg": 0.24824201, "policy/entropy_avg": 0.17230415, "val/ratio": 0.98949504, "val/ratio_var": 9.146e-05, "val/num_eos_tokens": 297, "lr": 9.95e-06, "episode": 1856, "epoch": 0.09373737, "global_step/max_steps": "116/1238", "percentage": "9.37%", "elapsed_time": "1h 17m 54s", "remaining_time": "12h 33m 32s"} +{"eps": 0, "objective/kl": 8.47564793, "objective/entropy": -68.10916901, "objective/non_score_reward": -0.42378241, "objective/rlhf_reward": 7.5176239, "objective/scores": 7.9375, "policy/approxkl_avg": 0.00078981, "policy/clipfrac_avg": 0.00639138, "loss/policy_avg": 0.13758664, "loss/value_avg": 1.60347593, "val/clipfrac_avg": 0.29291564, "policy/entropy_avg": 0.14587292, "val/ratio": 0.99050343, "val/ratio_var": 3.435e-05, "val/num_eos_tokens": 130, "lr": 9.95e-06, "episode": 1872, "epoch": 0.09454545, "global_step/max_steps": "117/1238", "percentage": "9.45%", "elapsed_time": "1h 18m 36s", "remaining_time": "12h 33m 13s"} +{"eps": 0, "objective/kl": 13.4442234, "objective/entropy": -64.47953033, "objective/non_score_reward": -0.67221117, "objective/rlhf_reward": 7.37075758, "objective/scores": 8.0625, "policy/approxkl_avg": 0.0009728, "policy/clipfrac_avg": 0.02348223, "loss/policy_avg": -0.01085886, "loss/value_avg": 0.8126936, "val/clipfrac_avg": 0.15926582, "policy/entropy_avg": 0.1738025, "val/ratio": 0.98877621, "val/ratio_var": 4.846e-05, "val/num_eos_tokens": 899, "lr": 9.95e-06, "episode": 1888, "epoch": 0.09535354, "global_step/max_steps": "118/1238", "percentage": "9.53%", "elapsed_time": "1h 19m 25s", "remaining_time": "12h 33m 47s"} +{"eps": 0, "objective/kl": 14.87445259, "objective/entropy": -142.52758789, "objective/non_score_reward": -0.74372262, "objective/rlhf_reward": 7.91643381, "objective/scores": 8.6875, "policy/approxkl_avg": 0.00139253, "policy/clipfrac_avg": 0.03409279, "loss/policy_avg": 0.05183769, "loss/value_avg": 0.73532605, "val/clipfrac_avg": 0.14925608, "policy/entropy_avg": 0.21674728, "val/ratio": 0.98913538, "val/ratio_var": 7.12e-05, "val/num_eos_tokens": 683, "lr": 9.94e-06, "episode": 1904, "epoch": 0.09616162, "global_step/max_steps": "119/1238", "percentage": "9.61%", "elapsed_time": "1h 20m 8s", "remaining_time": "12h 33m 32s"} +{"eps": 0, "objective/kl": 10.85112572, "objective/entropy": -30.89788246, "objective/non_score_reward": -0.54255623, "objective/rlhf_reward": 7.84367418, "objective/scores": 8.375, "policy/approxkl_avg": 0.0013286, "policy/clipfrac_avg": 0.02127312, "loss/policy_avg": 0.15797684, "loss/value_avg": 2.73953772, "val/clipfrac_avg": 0.16737978, "policy/entropy_avg": 0.20377636, "val/ratio": 0.98807883, "val/ratio_var": 5.169e-05, "val/num_eos_tokens": 345, "lr": 9.94e-06, "episode": 1920, "epoch": 0.0969697, "global_step/max_steps": "120/1238", "percentage": "9.69%", "elapsed_time": "1h 20m 44s", "remaining_time": "12h 32m 11s"} +{"eps": 0, "objective/kl": 14.51022911, "objective/entropy": -218.74690247, "objective/non_score_reward": -0.72551143, "objective/rlhf_reward": 8.21198845, "objective/scores": 8.9375, "policy/approxkl_avg": 0.00118375, "policy/clipfrac_avg": 0.04580179, "loss/policy_avg": 0.22806939, "loss/value_avg": 2.74325562, "val/clipfrac_avg": 0.14798504, "policy/entropy_avg": 0.18555427, "val/ratio": 0.99065679, "val/ratio_var": 7.052e-05, "val/num_eos_tokens": 350, "lr": 9.94e-06, "episode": 1936, "epoch": 0.09777778, "global_step/max_steps": "121/1238", "percentage": "9.77%", "elapsed_time": "1h 22m 18s", "remaining_time": "12h 39m 48s"} +{"eps": 0, "objective/kl": 10.68556023, "objective/entropy": -61.34413528, "objective/non_score_reward": -0.53427804, "objective/rlhf_reward": 9.78994083, "objective/scores": 10.3125, "policy/approxkl_avg": 0.00084754, "policy/clipfrac_avg": 0.00700628, "loss/policy_avg": -0.28502777, "loss/value_avg": 1.35619855, "val/clipfrac_avg": 0.12512645, "policy/entropy_avg": 0.19134903, "val/ratio": 0.98989773, "val/ratio_var": 6.246e-05, "val/num_eos_tokens": 19, "lr": 9.94e-06, "episode": 1952, "epoch": 0.09858586, "global_step/max_steps": "122/1238", "percentage": "9.85%", "elapsed_time": "1h 22m 51s", "remaining_time": "12h 37m 57s"} +{"eps": 0, "objective/kl": 11.46910667, "objective/entropy": -31.41351128, "objective/non_score_reward": -0.57345533, "objective/rlhf_reward": 10.35232639, "objective/scores": 10.9375, "policy/approxkl_avg": 0.00177501, "policy/clipfrac_avg": 0.01924217, "loss/policy_avg": -0.05578337, "loss/value_avg": 3.28669524, "val/clipfrac_avg": 0.1343042, "policy/entropy_avg": 0.23152542, "val/ratio": 0.98510623, "val/ratio_var": 0.00021594, "val/num_eos_tokens": 111, "lr": 9.94e-06, "episode": 1968, "epoch": 0.09939394, "global_step/max_steps": "123/1238", "percentage": "9.94%", "elapsed_time": "1h 23m 30s", "remaining_time": "12h 36m 57s"} +{"eps": 0, "objective/kl": 11.6789608, "objective/entropy": -41.85675812, "objective/non_score_reward": -0.58394802, "objective/rlhf_reward": 8.41605186, "objective/scores": 9.0, "policy/approxkl_avg": 0.00113457, "policy/clipfrac_avg": 0.00919476, "loss/policy_avg": 0.07291893, "loss/value_avg": 2.09426832, "val/clipfrac_avg": 0.20579089, "policy/entropy_avg": 0.2213707, "val/ratio": 0.98552614, "val/ratio_var": 0.00015781, "val/num_eos_tokens": 25, "lr": 9.93e-06, "episode": 1984, "epoch": 0.10020202, "global_step/max_steps": "124/1238", "percentage": "10.02%", "elapsed_time": "1h 24m 9s", "remaining_time": "12h 36m 6s"} +{"eps": 0, "objective/kl": 12.14737701, "objective/entropy": -38.61683655, "objective/non_score_reward": -0.60736895, "objective/rlhf_reward": 8.47856903, "objective/scores": 9.0625, "policy/approxkl_avg": 0.00104534, "policy/clipfrac_avg": 0.02015816, "loss/policy_avg": -0.20081276, "loss/value_avg": 0.97898233, "val/clipfrac_avg": 0.16441733, "policy/entropy_avg": 0.20670152, "val/ratio": 0.9876101, "val/ratio_var": 5.875e-05, "val/num_eos_tokens": 31, "lr": 9.93e-06, "episode": 2000, "epoch": 0.1010101, "global_step/max_steps": "125/1238", "percentage": "10.10%", "elapsed_time": "1h 25m 19s", "remaining_time": "12h 39m 40s"} +{"eps": 0, "objective/kl": 14.01747704, "objective/entropy": -137.78683472, "objective/non_score_reward": -0.70087385, "objective/rlhf_reward": 8.42803192, "objective/scores": 9.125, "policy/approxkl_avg": 0.00173886, "policy/clipfrac_avg": 0.02226019, "loss/policy_avg": -0.16343963, "loss/value_avg": 0.91922605, "val/clipfrac_avg": 0.23833248, "policy/entropy_avg": 0.17416728, "val/ratio": 0.98809421, "val/ratio_var": 0.00013718, "val/num_eos_tokens": 155, "lr": 9.93e-06, "episode": 2016, "epoch": 0.10181818, "global_step/max_steps": "126/1238", "percentage": "10.18%", "elapsed_time": "1h 26m 5s", "remaining_time": "12h 39m 44s"} +{"eps": 0, "objective/kl": 10.21295547, "objective/entropy": -77.51470947, "objective/non_score_reward": -0.51064777, "objective/rlhf_reward": 8.59872723, "objective/scores": 9.125, "policy/approxkl_avg": 0.00125953, "policy/clipfrac_avg": 0.01632132, "loss/policy_avg": -0.10418622, "loss/value_avg": 2.17414975, "val/clipfrac_avg": 0.32252413, "policy/entropy_avg": 0.16196728, "val/ratio": 0.9924022, "val/ratio_var": 0.00016108, "val/num_eos_tokens": 544, "lr": 9.93e-06, "episode": 2032, "epoch": 0.10262626, "global_step/max_steps": "127/1238", "percentage": "10.26%", "elapsed_time": "1h 26m 38s", "remaining_time": "12h 37m 52s"} +{"eps": 0, "objective/kl": 9.30160713, "objective/entropy": -71.0839386, "objective/non_score_reward": -0.46508032, "objective/rlhf_reward": 8.55445099, "objective/scores": 9.0, "policy/approxkl_avg": 0.00048919, "policy/clipfrac_avg": 0.00656355, "loss/policy_avg": 0.51390147, "loss/value_avg": 1.2218287, "val/clipfrac_avg": 0.1805044, "policy/entropy_avg": 0.09600616, "val/ratio": 0.99296564, "val/ratio_var": 3.002e-05, "val/num_eos_tokens": 269, "lr": 9.92e-06, "episode": 2048, "epoch": 0.10343434, "global_step/max_steps": "128/1238", "percentage": "10.34%", "elapsed_time": "1h 27m 9s", "remaining_time": "12h 35m 48s"} +{"eps": 0, "objective/kl": 10.985219, "objective/entropy": -131.27288818, "objective/non_score_reward": -0.54926097, "objective/rlhf_reward": 9.40386391, "objective/scores": 9.9375, "policy/approxkl_avg": 0.00067375, "policy/clipfrac_avg": 0.01042579, "loss/policy_avg": -0.42255756, "loss/value_avg": 0.80135423, "val/clipfrac_avg": 0.35581827, "policy/entropy_avg": 0.12466854, "val/ratio": 0.99168527, "val/ratio_var": 5.279e-05, "val/num_eos_tokens": 268, "lr": 9.92e-06, "episode": 2064, "epoch": 0.10424242, "global_step/max_steps": "129/1238", "percentage": "10.42%", "elapsed_time": "1h 27m 50s", "remaining_time": "12h 35m 10s"} +{"eps": 0, "objective/kl": 10.48823738, "objective/entropy": -134.74630737, "objective/non_score_reward": -0.5244118, "objective/rlhf_reward": 7.7724638, "objective/scores": 8.3125, "policy/approxkl_avg": 0.00090838, "policy/clipfrac_avg": 0.02275815, "loss/policy_avg": 0.66940272, "loss/value_avg": 2.38972569, "val/clipfrac_avg": 0.310045, "policy/entropy_avg": 0.17615557, "val/ratio": 0.99122906, "val/ratio_var": 4.814e-05, "val/num_eos_tokens": 43, "lr": 9.92e-06, "episode": 2080, "epoch": 0.10505051, "global_step/max_steps": "130/1238", "percentage": "10.50%", "elapsed_time": "1h 28m 36s", "remaining_time": "12h 35m 11s"} +{"eps": 0, "objective/kl": 8.39515781, "objective/entropy": -104.74658966, "objective/non_score_reward": -0.4197579, "objective/rlhf_reward": 8.46500778, "objective/scores": 8.875, "policy/approxkl_avg": 0.00169205, "policy/clipfrac_avg": 0.02493303, "loss/policy_avg": -0.26812419, "loss/value_avg": 1.85015368, "val/clipfrac_avg": 0.210372, "policy/entropy_avg": 0.15344143, "val/ratio": 0.99047959, "val/ratio_var": 0.0002193, "val/num_eos_tokens": 339, "lr": 9.92e-06, "episode": 2096, "epoch": 0.10585859, "global_step/max_steps": "131/1238", "percentage": "10.58%", "elapsed_time": "1h 29m 12s", "remaining_time": "12h 33m 46s"} +{"eps": 0, "objective/kl": 13.19589996, "objective/entropy": -64.35730743, "objective/non_score_reward": -0.65979493, "objective/rlhf_reward": 8.50817394, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00092213, "policy/clipfrac_avg": 0.00432593, "loss/policy_avg": -0.34324709, "loss/value_avg": 0.68972349, "val/clipfrac_avg": 0.07538644, "policy/entropy_avg": 0.18006849, "val/ratio": 0.98811603, "val/ratio_var": 6.756e-05, "val/num_eos_tokens": 210, "lr": 9.92e-06, "episode": 2112, "epoch": 0.10666667, "global_step/max_steps": "132/1238", "percentage": "10.66%", "elapsed_time": "1h 29m 49s", "remaining_time": "12h 32m 34s"} +{"eps": 0, "objective/kl": 10.79879189, "objective/entropy": -45.32240295, "objective/non_score_reward": -0.53993964, "objective/rlhf_reward": 7.83506012, "objective/scores": 8.375, "policy/approxkl_avg": 0.00352426, "policy/clipfrac_avg": 0.03283273, "loss/policy_avg": -0.0476741, "loss/value_avg": 0.76922834, "val/clipfrac_avg": 0.21977931, "policy/entropy_avg": 0.22318172, "val/ratio": 0.99331295, "val/ratio_var": 0.00219986, "val/num_eos_tokens": 493, "lr": 9.91e-06, "episode": 2128, "epoch": 0.10747475, "global_step/max_steps": "133/1238", "percentage": "10.74%", "elapsed_time": "1h 30m 31s", "remaining_time": "12h 32m 6s"} +{"eps": 0, "objective/kl": 12.16296577, "objective/entropy": -128.284729, "objective/non_score_reward": -0.60814834, "objective/rlhf_reward": 9.39575768, "objective/scores": 10.0, "policy/approxkl_avg": 0.0009851, "policy/clipfrac_avg": 0.02182108, "loss/policy_avg": -0.53228611, "loss/value_avg": 2.02564049, "val/clipfrac_avg": 0.30148917, "policy/entropy_avg": 0.12530065, "val/ratio": 0.99082434, "val/ratio_var": 0.00012694, "val/num_eos_tokens": 6, "lr": 9.91e-06, "episode": 2144, "epoch": 0.10828283, "global_step/max_steps": "134/1238", "percentage": "10.82%", "elapsed_time": "1h 31m 8s", "remaining_time": "12h 30m 54s"} +{"eps": 0, "objective/kl": 12.18817616, "objective/entropy": -63.13277054, "objective/non_score_reward": -0.60940886, "objective/rlhf_reward": 9.10934067, "objective/scores": 9.75, "policy/approxkl_avg": 0.00094032, "policy/clipfrac_avg": 0.01416954, "loss/policy_avg": -0.30308974, "loss/value_avg": 2.18315601, "val/clipfrac_avg": 0.24678542, "policy/entropy_avg": 0.15417695, "val/ratio": 0.99221158, "val/ratio_var": 0.00014833, "val/num_eos_tokens": 481, "lr": 9.91e-06, "episode": 2160, "epoch": 0.10909091, "global_step/max_steps": "135/1238", "percentage": "10.90%", "elapsed_time": "1h 31m 42s", "remaining_time": "12h 29m 17s"} +{"eps": 0, "objective/kl": 10.51830101, "objective/entropy": -103.97399902, "objective/non_score_reward": -0.52591509, "objective/rlhf_reward": 10.6811161, "objective/scores": 11.1875, "policy/approxkl_avg": 0.00075618, "policy/clipfrac_avg": 0.00300043, "loss/policy_avg": -0.85953057, "loss/value_avg": 2.59929299, "val/clipfrac_avg": 0.35328761, "policy/entropy_avg": 0.18250799, "val/ratio": 0.9901064, "val/ratio_var": 6.594e-05, "val/num_eos_tokens": 146, "lr": 9.91e-06, "episode": 2176, "epoch": 0.10989899, "global_step/max_steps": "136/1238", "percentage": "10.99%", "elapsed_time": "1h 32m 13s", "remaining_time": "12h 27m 21s"} +{"eps": 0, "objective/kl": 14.11211395, "objective/entropy": -123.25853729, "objective/non_score_reward": -0.70560569, "objective/rlhf_reward": 8.70455074, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00114218, "policy/clipfrac_avg": 0.02517348, "loss/policy_avg": 0.48323244, "loss/value_avg": 2.32854605, "val/clipfrac_avg": 0.17816152, "policy/entropy_avg": 0.13131452, "val/ratio": 0.99131262, "val/ratio_var": 5.519e-05, "val/num_eos_tokens": 190, "lr": 9.9e-06, "episode": 2192, "epoch": 0.11070707, "global_step/max_steps": "137/1238", "percentage": "11.07%", "elapsed_time": "1h 32m 53s", "remaining_time": "12h 26m 28s"} +{"eps": 0, "objective/kl": 14.98802948, "objective/entropy": -83.26674652, "objective/non_score_reward": -0.74940145, "objective/rlhf_reward": 8.88731766, "objective/scores": 9.625, "policy/approxkl_avg": 0.00327171, "policy/clipfrac_avg": 0.03480931, "loss/policy_avg": 0.0413128, "loss/value_avg": 1.22380471, "val/clipfrac_avg": 0.12297101, "policy/entropy_avg": 0.19169331, "val/ratio": 0.98759794, "val/ratio_var": 0.00023366, "val/num_eos_tokens": 458, "lr": 9.9e-06, "episode": 2208, "epoch": 0.11151515, "global_step/max_steps": "138/1238", "percentage": "11.15%", "elapsed_time": "1h 33m 50s", "remaining_time": "12h 28m 3s"} +{"eps": 0, "objective/kl": 10.46372604, "objective/entropy": -151.27680969, "objective/non_score_reward": -0.52318633, "objective/rlhf_reward": 9.84790802, "objective/scores": 10.375, "policy/approxkl_avg": 0.00155979, "policy/clipfrac_avg": 0.0335403, "loss/policy_avg": 0.10726684, "loss/value_avg": 1.2004472, "val/clipfrac_avg": 0.25066325, "policy/entropy_avg": 0.13694477, "val/ratio": 0.98885411, "val/ratio_var": 0.00017274, "val/num_eos_tokens": 461, "lr": 9.9e-06, "episode": 2224, "epoch": 0.11232323, "global_step/max_steps": "139/1238", "percentage": "11.23%", "elapsed_time": "1h 34m 36s", "remaining_time": "12h 27m 57s"} +{"eps": 0, "objective/kl": 12.37651348, "objective/entropy": -56.48613739, "objective/non_score_reward": -0.61882567, "objective/rlhf_reward": 7.49054956, "objective/scores": 8.125, "policy/approxkl_avg": 0.00091174, "policy/clipfrac_avg": 0.0137049, "loss/policy_avg": -0.13150293, "loss/value_avg": 1.11898661, "val/clipfrac_avg": 0.1203002, "policy/entropy_avg": 0.14188099, "val/ratio": 0.99168789, "val/ratio_var": 7.079e-05, "val/num_eos_tokens": 115, "lr": 9.89e-06, "episode": 2240, "epoch": 0.11313131, "global_step/max_steps": "140/1238", "percentage": "11.31%", "elapsed_time": "1h 35m 2s", "remaining_time": "12h 25m 25s"} +{"eps": 0, "objective/kl": 13.87989521, "objective/entropy": -100.3290329, "objective/non_score_reward": -0.69399476, "objective/rlhf_reward": 7.93491077, "objective/scores": 8.625, "policy/approxkl_avg": 0.00085541, "policy/clipfrac_avg": 0.01539596, "loss/policy_avg": -0.03175282, "loss/value_avg": 0.97981393, "val/clipfrac_avg": 0.11368512, "policy/entropy_avg": 0.14811945, "val/ratio": 0.99018753, "val/ratio_var": 6.502e-05, "val/num_eos_tokens": 926, "lr": 9.89e-06, "episode": 2256, "epoch": 0.11393939, "global_step/max_steps": "141/1238", "percentage": "11.39%", "elapsed_time": "1h 35m 50s", "remaining_time": "12h 25m 40s"} +{"eps": 0, "objective/kl": 14.76631927, "objective/entropy": -61.57604599, "objective/non_score_reward": -0.73831606, "objective/rlhf_reward": 7.91012144, "objective/scores": 8.625, "policy/approxkl_avg": 0.00127965, "policy/clipfrac_avg": 0.01897471, "loss/policy_avg": 0.16777146, "loss/value_avg": 1.33881998, "val/clipfrac_avg": 0.10001248, "policy/entropy_avg": 0.15843773, "val/ratio": 0.98931211, "val/ratio_var": 4.403e-05, "val/num_eos_tokens": 250, "lr": 9.89e-06, "episode": 2272, "epoch": 0.11474747, "global_step/max_steps": "142/1238", "percentage": "11.47%", "elapsed_time": "1h 36m 26s", "remaining_time": "12h 24m 21s"} +{"eps": 0, "objective/kl": 16.06870079, "objective/entropy": -124.78637695, "objective/non_score_reward": -0.80343509, "objective/rlhf_reward": 8.33718967, "objective/scores": 9.125, "policy/approxkl_avg": 0.00096659, "policy/clipfrac_avg": 0.02759075, "loss/policy_avg": 0.28672373, "loss/value_avg": 2.62498045, "val/clipfrac_avg": 0.26182559, "policy/entropy_avg": 0.26673293, "val/ratio": 0.99185598, "val/ratio_var": 4.805e-05, "val/num_eos_tokens": 87, "lr": 9.89e-06, "episode": 2288, "epoch": 0.11555556, "global_step/max_steps": "143/1238", "percentage": "11.55%", "elapsed_time": "1h 37m 8s", "remaining_time": "12h 23m 49s"} +{"eps": 0, "objective/kl": 14.85019588, "objective/entropy": -264.25631714, "objective/non_score_reward": -0.74250984, "objective/rlhf_reward": 6.92936516, "objective/scores": 7.6875, "policy/approxkl_avg": 0.00073131, "policy/clipfrac_avg": 0.03529049, "loss/policy_avg": 0.61260206, "loss/value_avg": 1.57997715, "val/clipfrac_avg": 0.1675826, "policy/entropy_avg": 0.17197454, "val/ratio": 0.9931252, "val/ratio_var": 4.072e-05, "val/num_eos_tokens": 104, "lr": 9.88e-06, "episode": 2304, "epoch": 0.11636364, "global_step/max_steps": "144/1238", "percentage": "11.63%", "elapsed_time": "1h 38m 8s", "remaining_time": "12h 25m 33s"} +{"eps": 0, "objective/kl": 17.77285194, "objective/entropy": -43.33739853, "objective/non_score_reward": -0.88864267, "objective/rlhf_reward": 7.29690456, "objective/scores": 8.1875, "policy/approxkl_avg": 0.00089216, "policy/clipfrac_avg": 0.01221778, "loss/policy_avg": 0.11925016, "loss/value_avg": 1.10499001, "val/clipfrac_avg": 0.22841123, "policy/entropy_avg": 0.17539597, "val/ratio": 0.99085367, "val/ratio_var": 6.429e-05, "val/num_eos_tokens": 268, "lr": 9.88e-06, "episode": 2320, "epoch": 0.11717172, "global_step/max_steps": "145/1238", "percentage": "11.71%", "elapsed_time": "1h 38m 37s", "remaining_time": "12h 23m 24s"} +{"eps": 0, "objective/kl": 13.6993618, "objective/entropy": -95.03627777, "objective/non_score_reward": -0.68496811, "objective/rlhf_reward": 10.24471855, "objective/scores": 10.9375, "policy/approxkl_avg": 0.01541473, "policy/clipfrac_avg": 0.0417903, "loss/policy_avg": 0.08043037, "loss/value_avg": 1.52159548, "val/clipfrac_avg": 0.18847555, "policy/entropy_avg": 0.11050856, "val/ratio": 0.99070859, "val/ratio_var": 0.00013218, "val/num_eos_tokens": 473, "lr": 9.88e-06, "episode": 2336, "epoch": 0.1179798, "global_step/max_steps": "146/1238", "percentage": "11.79%", "elapsed_time": "1h 39m 19s", "remaining_time": "12h 22m 57s"} +{"eps": 0, "objective/kl": 15.63423824, "objective/entropy": -81.37150574, "objective/non_score_reward": -0.78171194, "objective/rlhf_reward": 10.05422497, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00119868, "policy/clipfrac_avg": 0.00917242, "loss/policy_avg": -0.09133088, "loss/value_avg": 1.02377176, "val/clipfrac_avg": 0.20073412, "policy/entropy_avg": 0.1481331, "val/ratio": 0.99322039, "val/ratio_var": 0.00023358, "val/num_eos_tokens": 175, "lr": 9.87e-06, "episode": 2352, "epoch": 0.11878788, "global_step/max_steps": "147/1238", "percentage": "11.87%", "elapsed_time": "1h 39m 58s", "remaining_time": "12h 22m 0s"} +{"eps": 0, "objective/kl": 18.75476456, "objective/entropy": -58.46908569, "objective/non_score_reward": -0.9377383, "objective/rlhf_reward": 7.87671471, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00123355, "policy/clipfrac_avg": 0.01598468, "loss/policy_avg": -0.00169966, "loss/value_avg": 0.85997486, "val/clipfrac_avg": 0.23399399, "policy/entropy_avg": 0.21374321, "val/ratio": 0.98767292, "val/ratio_var": 0.00011754, "val/num_eos_tokens": 339, "lr": 9.87e-06, "episode": 2368, "epoch": 0.11959596, "global_step/max_steps": "148/1238", "percentage": "11.95%", "elapsed_time": "1h 40m 38s", "remaining_time": "12h 21m 12s"} +{"eps": 0, "objective/kl": 17.04312134, "objective/entropy": -153.42767334, "objective/non_score_reward": -0.85215604, "objective/rlhf_reward": 7.32753134, "objective/scores": 8.1875, "policy/approxkl_avg": 0.00074524, "policy/clipfrac_avg": 0.00415482, "loss/policy_avg": -0.31326699, "loss/value_avg": 0.74713045, "val/clipfrac_avg": 0.2773369, "policy/entropy_avg": 0.13955671, "val/ratio": 0.99115872, "val/ratio_var": 4.037e-05, "val/num_eos_tokens": 543, "lr": 9.87e-06, "episode": 2384, "epoch": 0.12040404, "global_step/max_steps": "149/1238", "percentage": "12.04%", "elapsed_time": "1h 41m 28s", "remaining_time": "12h 21m 38s"} +{"eps": 0, "objective/kl": 15.88914585, "objective/entropy": -134.2124939, "objective/non_score_reward": -0.79445732, "objective/rlhf_reward": 8.15476131, "objective/scores": 8.9375, "policy/approxkl_avg": 0.00086212, "policy/clipfrac_avg": 0.02879198, "loss/policy_avg": 0.58235478, "loss/value_avg": 2.28339314, "val/clipfrac_avg": 0.35077244, "policy/entropy_avg": 0.16280103, "val/ratio": 0.99147731, "val/ratio_var": 6.433e-05, "val/num_eos_tokens": 742, "lr": 9.87e-06, "episode": 2400, "epoch": 0.12121212, "global_step/max_steps": "150/1238", "percentage": "12.12%", "elapsed_time": "1h 42m 6s", "remaining_time": "12h 20m 38s"} +{"eps": 0, "objective/kl": 16.73929596, "objective/entropy": -73.51275635, "objective/non_score_reward": -0.83696473, "objective/rlhf_reward": 10.03412914, "objective/scores": 10.875, "policy/approxkl_avg": 0.00080296, "policy/clipfrac_avg": 0.00789409, "loss/policy_avg": 0.00687573, "loss/value_avg": 0.53641939, "val/clipfrac_avg": 0.25386879, "policy/entropy_avg": 0.1546452, "val/ratio": 0.99045098, "val/ratio_var": 2.984e-05, "val/num_eos_tokens": 33, "lr": 9.86e-06, "episode": 2416, "epoch": 0.1220202, "global_step/max_steps": "151/1238", "percentage": "12.20%", "elapsed_time": "1h 42m 52s", "remaining_time": "12h 20m 32s"} +{"eps": 0, "objective/kl": 14.28682137, "objective/entropy": -136.72979736, "objective/non_score_reward": -0.7143411, "objective/rlhf_reward": 9.17237759, "objective/scores": 9.875, "policy/approxkl_avg": 0.00117722, "policy/clipfrac_avg": 0.01603562, "loss/policy_avg": 0.46432221, "loss/value_avg": 2.50265527, "val/clipfrac_avg": 0.28627664, "policy/entropy_avg": 0.13455468, "val/ratio": 0.99006283, "val/ratio_var": 7.127e-05, "val/num_eos_tokens": 619, "lr": 9.86e-06, "episode": 2432, "epoch": 0.12282828, "global_step/max_steps": "152/1238", "percentage": "12.28%", "elapsed_time": "1h 43m 25s", "remaining_time": "12h 18m 57s"} +{"eps": 0, "objective/kl": 16.09568214, "objective/entropy": -138.69039917, "objective/non_score_reward": -0.80478412, "objective/rlhf_reward": 8.84365368, "objective/scores": 9.625, "policy/approxkl_avg": 0.0012001, "policy/clipfrac_avg": 0.01806447, "loss/policy_avg": 0.09378673, "loss/value_avg": 3.24774027, "val/clipfrac_avg": 0.29855174, "policy/entropy_avg": 0.14937532, "val/ratio": 0.98930687, "val/ratio_var": 6.622e-05, "val/num_eos_tokens": 433, "lr": 9.86e-06, "episode": 2448, "epoch": 0.12363636, "global_step/max_steps": "153/1238", "percentage": "12.36%", "elapsed_time": "1h 44m 10s", "remaining_time": "12h 18m 44s"} +{"eps": 0, "objective/kl": 19.26645279, "objective/entropy": -80.84510803, "objective/non_score_reward": -0.96332264, "objective/rlhf_reward": 8.34136391, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00077892, "policy/clipfrac_avg": 0.00622265, "loss/policy_avg": -0.35949445, "loss/value_avg": 2.61572528, "val/clipfrac_avg": 0.14149022, "policy/entropy_avg": 0.12923431, "val/ratio": 0.99127632, "val/ratio_var": 5.006e-05, "val/num_eos_tokens": 65, "lr": 9.85e-06, "episode": 2464, "epoch": 0.12444444, "global_step/max_steps": "154/1238", "percentage": "12.44%", "elapsed_time": "1h 44m 59s", "remaining_time": "12h 19m 0s"} +{"eps": 0, "objective/kl": 16.32784462, "objective/entropy": -114.74142456, "objective/non_score_reward": -0.8163923, "objective/rlhf_reward": 9.56642056, "objective/scores": 10.375, "policy/approxkl_avg": 0.00104653, "policy/clipfrac_avg": 0.01326069, "loss/policy_avg": 0.13503911, "loss/value_avg": 1.30539858, "val/clipfrac_avg": 0.12556566, "policy/entropy_avg": 0.13961029, "val/ratio": 0.99143469, "val/ratio_var": 8.752e-05, "val/num_eos_tokens": 24, "lr": 9.85e-06, "episode": 2480, "epoch": 0.12525253, "global_step/max_steps": "155/1238", "percentage": "12.52%", "elapsed_time": "1h 45m 40s", "remaining_time": "12h 18m 20s"} +{"eps": 0, "objective/kl": 14.01751709, "objective/entropy": -51.69160461, "objective/non_score_reward": -0.70087588, "objective/rlhf_reward": 9.08428001, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0020043, "policy/clipfrac_avg": 0.02591789, "loss/policy_avg": -0.28625339, "loss/value_avg": 1.49721384, "val/clipfrac_avg": 0.07307614, "policy/entropy_avg": 0.18289995, "val/ratio": 0.98842692, "val/ratio_var": 0.00010893, "val/num_eos_tokens": 89, "lr": 9.85e-06, "episode": 2496, "epoch": 0.12606061, "global_step/max_steps": "156/1238", "percentage": "12.60%", "elapsed_time": "1h 46m 14s", "remaining_time": "12h 16m 52s"} +{"eps": 0, "objective/kl": 18.98889542, "objective/entropy": -38.66559601, "objective/non_score_reward": -0.94944483, "objective/rlhf_reward": 8.07008648, "objective/scores": 9.0, "policy/approxkl_avg": 0.00497173, "policy/clipfrac_avg": 0.01767452, "loss/policy_avg": 0.08509958, "loss/value_avg": 1.0140475, "val/clipfrac_avg": 0.1631356, "policy/entropy_avg": 0.19965363, "val/ratio": 0.99060422, "val/ratio_var": 0.00015354, "val/num_eos_tokens": 85, "lr": 9.84e-06, "episode": 2512, "epoch": 0.12686869, "global_step/max_steps": "157/1238", "percentage": "12.68%", "elapsed_time": "1h 46m 49s", "remaining_time": "12h 15m 33s"} +{"eps": 0, "objective/kl": 14.48059082, "objective/entropy": -81.0584259, "objective/non_score_reward": -0.72402954, "objective/rlhf_reward": 9.06893921, "objective/scores": 9.8125, "policy/approxkl_avg": 0.00112426, "policy/clipfrac_avg": 0.03127669, "loss/policy_avg": -0.30014685, "loss/value_avg": 2.4983511, "val/clipfrac_avg": 0.15103458, "policy/entropy_avg": 0.18725228, "val/ratio": 0.99331403, "val/ratio_var": 0.00010325, "val/num_eos_tokens": 3, "lr": 9.84e-06, "episode": 2528, "epoch": 0.12767677, "global_step/max_steps": "158/1238", "percentage": "12.76%", "elapsed_time": "1h 47m 23s", "remaining_time": "12h 14m 2s"} +{"eps": 0, "objective/kl": 18.89221954, "objective/entropy": -75.23670197, "objective/non_score_reward": -0.94461095, "objective/rlhf_reward": 9.1608572, "objective/scores": 10.125, "policy/approxkl_avg": 0.00531356, "policy/clipfrac_avg": 0.01770968, "loss/policy_avg": 0.03238319, "loss/value_avg": 0.92456979, "val/clipfrac_avg": 0.09926829, "policy/entropy_avg": 0.26306152, "val/ratio": 0.98458701, "val/ratio_var": 0.00043843, "val/num_eos_tokens": 214, "lr": 9.84e-06, "episode": 2544, "epoch": 0.12848485, "global_step/max_steps": "159/1238", "percentage": "12.84%", "elapsed_time": "1h 48m 4s", "remaining_time": "12h 13m 23s"} +{"eps": 0, "objective/kl": 14.31398582, "objective/entropy": -58.01062775, "objective/non_score_reward": -0.7156992, "objective/rlhf_reward": 7.6671133, "objective/scores": 8.375, "policy/approxkl_avg": 0.00088902, "policy/clipfrac_avg": 0.00987298, "loss/policy_avg": -0.00330285, "loss/value_avg": 1.93828678, "val/clipfrac_avg": 0.11537375, "policy/entropy_avg": 0.19803333, "val/ratio": 0.9886167, "val/ratio_var": 5.669e-05, "val/num_eos_tokens": 85, "lr": 9.83e-06, "episode": 2560, "epoch": 0.12929293, "global_step/max_steps": "160/1238", "percentage": "12.92%", "elapsed_time": "1h 48m 37s", "remaining_time": "12h 11m 53s"} +{"eps": 0, "objective/kl": 14.48710632, "objective/entropy": -105.31230164, "objective/non_score_reward": -0.72435534, "objective/rlhf_reward": 9.4943943, "objective/scores": 10.25, "policy/approxkl_avg": 0.00084578, "policy/clipfrac_avg": 0.01930004, "loss/policy_avg": 0.15009576, "loss/value_avg": 1.30364227, "val/clipfrac_avg": 0.2028054, "policy/entropy_avg": 0.16261721, "val/ratio": 0.99087411, "val/ratio_var": 8.265e-05, "val/num_eos_tokens": 333, "lr": 9.83e-06, "episode": 2576, "epoch": 0.13010101, "global_step/max_steps": "161/1238", "percentage": "13.00%", "elapsed_time": "1h 49m 11s", "remaining_time": "12h 10m 25s"} +{"eps": 0, "objective/kl": 16.28435516, "objective/entropy": -119.47211456, "objective/non_score_reward": -0.81421781, "objective/rlhf_reward": 6.92796946, "objective/scores": 7.75, "policy/approxkl_avg": 0.00120127, "policy/clipfrac_avg": 0.03371775, "loss/policy_avg": 0.75363553, "loss/value_avg": 2.16429925, "val/clipfrac_avg": 0.36582974, "policy/entropy_avg": 0.20034599, "val/ratio": 0.9901647, "val/ratio_var": 9.402e-05, "val/num_eos_tokens": 737, "lr": 9.83e-06, "episode": 2592, "epoch": 0.13090909, "global_step/max_steps": "162/1238", "percentage": "13.09%", "elapsed_time": "1h 49m 54s", "remaining_time": "12h 10m 1s"} +{"eps": 0, "objective/kl": 17.1247673, "objective/entropy": -88.83195496, "objective/non_score_reward": -0.85623837, "objective/rlhf_reward": 9.32735634, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00073947, "policy/clipfrac_avg": 0.01986416, "loss/policy_avg": -0.27083319, "loss/value_avg": 1.6828413, "val/clipfrac_avg": 0.30267167, "policy/entropy_avg": 0.13005447, "val/ratio": 0.9908427, "val/ratio_var": 6.519e-05, "val/num_eos_tokens": 512, "lr": 9.82e-06, "episode": 2608, "epoch": 0.13171717, "global_step/max_steps": "163/1238", "percentage": "13.17%", "elapsed_time": "1h 50m 30s", "remaining_time": "12h 8m 49s"} +{"eps": 0, "objective/kl": 15.06971359, "objective/entropy": -210.48764038, "objective/non_score_reward": -0.75348568, "objective/rlhf_reward": 7.66838932, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00092566, "policy/clipfrac_avg": 0.02412328, "loss/policy_avg": -0.37486351, "loss/value_avg": 2.39401269, "val/clipfrac_avg": 0.14958948, "policy/entropy_avg": 0.16263807, "val/ratio": 0.9890939, "val/ratio_var": 0.00011626, "val/num_eos_tokens": 676, "lr": 9.82e-06, "episode": 2624, "epoch": 0.13252525, "global_step/max_steps": "164/1238", "percentage": "13.25%", "elapsed_time": "1h 52m 14s", "remaining_time": "12h 15m 5s"} +{"eps": 0, "objective/kl": 16.53316498, "objective/entropy": -53.14797592, "objective/non_score_reward": -0.82665825, "objective/rlhf_reward": 8.817873, "objective/scores": 9.625, "policy/approxkl_avg": 0.00091104, "policy/clipfrac_avg": 0.00571254, "loss/policy_avg": -0.30216762, "loss/value_avg": 0.79702461, "val/clipfrac_avg": 0.15537137, "policy/entropy_avg": 0.17499542, "val/ratio": 0.98888135, "val/ratio_var": 3.098e-05, "val/num_eos_tokens": 536, "lr": 9.82e-06, "episode": 2640, "epoch": 0.13333333, "global_step/max_steps": "165/1238", "percentage": "13.33%", "elapsed_time": "1h 52m 56s", "remaining_time": "12h 14m 29s"} +{"eps": 0, "objective/kl": 12.42782974, "objective/entropy": -61.10470963, "objective/non_score_reward": -0.62139153, "objective/rlhf_reward": 8.55438995, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00088505, "policy/clipfrac_avg": 0.01336047, "loss/policy_avg": 0.10213344, "loss/value_avg": 0.98538393, "val/clipfrac_avg": 0.25333312, "policy/entropy_avg": 0.16296458, "val/ratio": 0.99043322, "val/ratio_var": 7.649e-05, "val/num_eos_tokens": 73, "lr": 9.81e-06, "episode": 2656, "epoch": 0.13414141, "global_step/max_steps": "166/1238", "percentage": "13.41%", "elapsed_time": "1h 53m 33s", "remaining_time": "12h 13m 23s"} +{"eps": 0, "objective/kl": 14.755229, "objective/entropy": -38.80767059, "objective/non_score_reward": -0.7377615, "objective/rlhf_reward": 7.59426975, "objective/scores": 8.3125, "policy/approxkl_avg": 0.00331338, "policy/clipfrac_avg": 0.01232538, "loss/policy_avg": 0.51100564, "loss/value_avg": 1.07730675, "val/clipfrac_avg": 0.16639975, "policy/entropy_avg": 0.20802689, "val/ratio": 0.98780167, "val/ratio_var": 5.612e-05, "val/num_eos_tokens": 280, "lr": 9.81e-06, "episode": 2672, "epoch": 0.13494949, "global_step/max_steps": "167/1238", "percentage": "13.49%", "elapsed_time": "1h 54m 5s", "remaining_time": "12h 11m 42s"} +{"eps": 0, "objective/kl": 14.00096035, "objective/entropy": -45.07624817, "objective/non_score_reward": -0.70004803, "objective/rlhf_reward": 9.37026501, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00280546, "policy/clipfrac_avg": 0.01768944, "loss/policy_avg": -0.06541467, "loss/value_avg": 1.88665485, "val/clipfrac_avg": 0.40120679, "policy/entropy_avg": 0.20137405, "val/ratio": 0.98430705, "val/ratio_var": 0.00026259, "val/num_eos_tokens": 219, "lr": 9.8e-06, "episode": 2688, "epoch": 0.13575758, "global_step/max_steps": "168/1238", "percentage": "13.57%", "elapsed_time": "1h 54m 33s", "remaining_time": "12h 9m 40s"} +{"eps": 0, "objective/kl": 12.76452065, "objective/entropy": -29.56934547, "objective/non_score_reward": -0.63822603, "objective/rlhf_reward": 9.59614944, "objective/scores": 10.25, "policy/approxkl_avg": 0.00262776, "policy/clipfrac_avg": 0.02065247, "loss/policy_avg": -0.26235154, "loss/value_avg": 0.93611538, "val/clipfrac_avg": 0.07669641, "policy/entropy_avg": 0.18619645, "val/ratio": 0.98801887, "val/ratio_var": 0.00037325, "val/num_eos_tokens": 114, "lr": 9.8e-06, "episode": 2704, "epoch": 0.13656566, "global_step/max_steps": "169/1238", "percentage": "13.65%", "elapsed_time": "1h 55m 3s", "remaining_time": "12h 7m 45s"} +{"eps": 0, "objective/kl": 14.56460476, "objective/entropy": -115.31767273, "objective/non_score_reward": -0.72823024, "objective/rlhf_reward": 8.54130173, "objective/scores": 9.25, "policy/approxkl_avg": 0.00097976, "policy/clipfrac_avg": 0.01354642, "loss/policy_avg": 0.00365169, "loss/value_avg": 0.78085345, "val/clipfrac_avg": 0.13324237, "policy/entropy_avg": 0.17569169, "val/ratio": 0.98803592, "val/ratio_var": 5.695e-05, "val/num_eos_tokens": 24, "lr": 9.8e-06, "episode": 2720, "epoch": 0.13737374, "global_step/max_steps": "170/1238", "percentage": "13.73%", "elapsed_time": "1h 55m 52s", "remaining_time": "12h 8m 0s"} +{"eps": 0, "objective/kl": 17.0995388, "objective/entropy": -65.40606689, "objective/non_score_reward": -0.85497701, "objective/rlhf_reward": 8.24658585, "objective/scores": 9.125, "policy/approxkl_avg": 0.00091733, "policy/clipfrac_avg": 0.01118012, "loss/policy_avg": -0.31284106, "loss/value_avg": 0.69498456, "val/clipfrac_avg": 0.09039228, "policy/entropy_avg": 0.17095184, "val/ratio": 0.98899561, "val/ratio_var": 5.042e-05, "val/num_eos_tokens": 391, "lr": 9.79e-06, "episode": 2736, "epoch": 0.13818182, "global_step/max_steps": "171/1238", "percentage": "13.81%", "elapsed_time": "1h 56m 29s", "remaining_time": "12h 6m 51s"} +{"eps": 0, "objective/kl": 14.35270023, "objective/entropy": -149.44508362, "objective/non_score_reward": -0.71763504, "objective/rlhf_reward": 7.44642735, "objective/scores": 8.1875, "policy/approxkl_avg": 0.00075134, "policy/clipfrac_avg": 0.01495585, "loss/policy_avg": -0.06095901, "loss/value_avg": 0.68172181, "val/clipfrac_avg": 0.3020364, "policy/entropy_avg": 0.1538682, "val/ratio": 0.99014115, "val/ratio_var": 5.243e-05, "val/num_eos_tokens": 70, "lr": 9.79e-06, "episode": 2752, "epoch": 0.1389899, "global_step/max_steps": "172/1238", "percentage": "13.89%", "elapsed_time": "1h 57m 10s", "remaining_time": "12h 6m 9s"} +{"eps": 0, "objective/kl": 18.06300354, "objective/entropy": -103.21472168, "objective/non_score_reward": -0.90315026, "objective/rlhf_reward": 10.32341194, "objective/scores": 11.25, "policy/approxkl_avg": 0.00116485, "policy/clipfrac_avg": 0.02175794, "loss/policy_avg": -0.90737212, "loss/value_avg": 3.30962586, "val/clipfrac_avg": 0.3607612, "policy/entropy_avg": 0.1891931, "val/ratio": 0.98899281, "val/ratio_var": 0.00014562, "val/num_eos_tokens": 583, "lr": 9.79e-06, "episode": 2768, "epoch": 0.13979798, "global_step/max_steps": "173/1238", "percentage": "13.97%", "elapsed_time": "1h 57m 46s", "remaining_time": "12h 5m 1s"} +{"eps": 0, "objective/kl": 15.56573486, "objective/entropy": -144.74966431, "objective/non_score_reward": -0.7782867, "objective/rlhf_reward": 7.42288542, "objective/scores": 8.1875, "policy/approxkl_avg": 0.00080356, "policy/clipfrac_avg": 0.03028783, "loss/policy_avg": 0.97713125, "loss/value_avg": 3.14011908, "val/clipfrac_avg": 0.32685003, "policy/entropy_avg": 0.13976645, "val/ratio": 0.99073392, "val/ratio_var": 4.144e-05, "val/num_eos_tokens": 455, "lr": 9.78e-06, "episode": 2784, "epoch": 0.14060606, "global_step/max_steps": "174/1238", "percentage": "14.05%", "elapsed_time": "1h 58m 33s", "remaining_time": "12h 4m 56s"} +{"eps": 0, "objective/kl": 14.40029716, "objective/entropy": -106.90374756, "objective/non_score_reward": -0.72001487, "objective/rlhf_reward": 7.99873543, "objective/scores": 8.75, "policy/approxkl_avg": 0.00098683, "policy/clipfrac_avg": 0.0092371, "loss/policy_avg": 0.28167734, "loss/value_avg": 1.64073133, "val/clipfrac_avg": 0.19442004, "policy/entropy_avg": 0.16825366, "val/ratio": 0.98990917, "val/ratio_var": 9.542e-05, "val/num_eos_tokens": 59, "lr": 9.78e-06, "episode": 2800, "epoch": 0.14141414, "global_step/max_steps": "175/1238", "percentage": "14.14%", "elapsed_time": "1h 59m 4s", "remaining_time": "12h 3m 15s"} +{"eps": 0, "objective/kl": 9.65359306, "objective/entropy": -60.39266205, "objective/non_score_reward": -0.48267969, "objective/rlhf_reward": 9.05247593, "objective/scores": 9.5625, "policy/approxkl_avg": 0.00107499, "policy/clipfrac_avg": 0.00982856, "loss/policy_avg": -0.15210214, "loss/value_avg": 1.22738338, "val/clipfrac_avg": 0.11848569, "policy/entropy_avg": 0.16010571, "val/ratio": 0.9885416, "val/ratio_var": 5.873e-05, "val/num_eos_tokens": 58, "lr": 9.77e-06, "episode": 2816, "epoch": 0.14222222, "global_step/max_steps": "176/1238", "percentage": "14.22%", "elapsed_time": "1h 59m 42s", "remaining_time": "12h 2m 21s"} +{"eps": 0, "objective/kl": 11.46190453, "objective/entropy": -69.82381439, "objective/non_score_reward": -0.57309526, "objective/rlhf_reward": 11.04409218, "objective/scores": 11.625, "policy/approxkl_avg": 0.00132581, "policy/clipfrac_avg": 0.02767137, "loss/policy_avg": 0.1739445, "loss/value_avg": 3.3872366, "val/clipfrac_avg": 0.25187913, "policy/entropy_avg": 0.15888667, "val/ratio": 0.99169987, "val/ratio_var": 7.523e-05, "val/num_eos_tokens": 28, "lr": 9.77e-06, "episode": 2832, "epoch": 0.1430303, "global_step/max_steps": "177/1238", "percentage": "14.30%", "elapsed_time": "2h 0m 15s", "remaining_time": "12h 0m 51s"} +{"eps": 0, "objective/kl": 16.81972313, "objective/entropy": -124.40007019, "objective/non_score_reward": -0.84098613, "objective/rlhf_reward": 10.6394825, "objective/scores": 11.5, "policy/approxkl_avg": 0.00101324, "policy/clipfrac_avg": 0.0088144, "loss/policy_avg": -0.12190956, "loss/value_avg": 1.3829248, "val/clipfrac_avg": 0.22931249, "policy/entropy_avg": 0.15328598, "val/ratio": 0.98888659, "val/ratio_var": 7.728e-05, "val/num_eos_tokens": 372, "lr": 9.77e-06, "episode": 2848, "epoch": 0.14383838, "global_step/max_steps": "178/1238", "percentage": "14.38%", "elapsed_time": "2h 1m 4s", "remaining_time": "12h 0m 59s"} +{"eps": 0, "objective/kl": 12.1277647, "objective/entropy": -97.34057617, "objective/non_score_reward": -0.60638821, "objective/rlhf_reward": 8.14165878, "objective/scores": 8.75, "policy/approxkl_avg": 0.00083688, "policy/clipfrac_avg": 0.01260412, "loss/policy_avg": -0.36637282, "loss/value_avg": 1.25808263, "val/clipfrac_avg": 0.08666436, "policy/entropy_avg": 0.1529665, "val/ratio": 0.98959756, "val/ratio_var": 6.254e-05, "val/num_eos_tokens": 53, "lr": 9.76e-06, "episode": 2864, "epoch": 0.14464646, "global_step/max_steps": "179/1238", "percentage": "14.46%", "elapsed_time": "2h 1m 49s", "remaining_time": "12h 0m 45s"} +{"eps": 0, "objective/kl": 12.38321686, "objective/entropy": -92.20721436, "objective/non_score_reward": -0.61916083, "objective/rlhf_reward": 10.09177685, "objective/scores": 10.6875, "policy/approxkl_avg": 0.0010203, "policy/clipfrac_avg": 0.02049468, "loss/policy_avg": -0.7018677, "loss/value_avg": 2.8395381, "val/clipfrac_avg": 0.19666892, "policy/entropy_avg": 0.14533499, "val/ratio": 0.9898504, "val/ratio_var": 8.984e-05, "val/num_eos_tokens": 4, "lr": 9.76e-06, "episode": 2880, "epoch": 0.14545455, "global_step/max_steps": "180/1238", "percentage": "14.54%", "elapsed_time": "2h 2m 25s", "remaining_time": "11h 59m 37s"} +{"eps": 0, "objective/kl": 12.26214027, "objective/entropy": -76.44052124, "objective/non_score_reward": -0.61310703, "objective/rlhf_reward": 9.07829857, "objective/scores": 9.6875, "policy/approxkl_avg": 0.00106979, "policy/clipfrac_avg": 0.02236181, "loss/policy_avg": -0.12757133, "loss/value_avg": 2.16737485, "val/clipfrac_avg": 0.3174876, "policy/entropy_avg": 0.15213966, "val/ratio": 0.99031365, "val/ratio_var": 8.45e-05, "val/num_eos_tokens": 158, "lr": 9.75e-06, "episode": 2896, "epoch": 0.14626263, "global_step/max_steps": "181/1238", "percentage": "14.62%", "elapsed_time": "2h 2m 57s", "remaining_time": "11h 58m 5s"} +{"eps": 0, "objective/kl": 12.12323093, "objective/entropy": -149.43963623, "objective/non_score_reward": -0.60616159, "objective/rlhf_reward": 10.07352543, "objective/scores": 10.6875, "policy/approxkl_avg": 0.00086506, "policy/clipfrac_avg": 0.02972309, "loss/policy_avg": -0.03054467, "loss/value_avg": 1.8187933, "val/clipfrac_avg": 0.12663344, "policy/entropy_avg": 0.14513269, "val/ratio": 0.99405289, "val/ratio_var": 9.983e-05, "val/num_eos_tokens": 192, "lr": 9.75e-06, "episode": 2912, "epoch": 0.14707071, "global_step/max_steps": "182/1238", "percentage": "14.70%", "elapsed_time": "2h 3m 31s", "remaining_time": "11h 56m 41s"} +{"eps": 0, "objective/kl": 15.38311577, "objective/entropy": -50.62373352, "objective/non_score_reward": -0.76915574, "objective/rlhf_reward": 9.64100075, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00447791, "policy/clipfrac_avg": 0.05062967, "loss/policy_avg": 0.08868739, "loss/value_avg": 1.87852716, "val/clipfrac_avg": 0.20940411, "policy/entropy_avg": 0.2141428, "val/ratio": 0.98433965, "val/ratio_var": 0.00037895, "val/num_eos_tokens": 2, "lr": 9.75e-06, "episode": 2928, "epoch": 0.14787879, "global_step/max_steps": "183/1238", "percentage": "14.78%", "elapsed_time": "2h 3m 58s", "remaining_time": "11h 54m 45s"} +{"eps": 0, "objective/kl": 15.15971375, "objective/entropy": -82.41788483, "objective/non_score_reward": -0.75798571, "objective/rlhf_reward": 7.96857643, "objective/scores": 8.75, "policy/approxkl_avg": 0.00107793, "policy/clipfrac_avg": 0.01132072, "loss/policy_avg": -0.36278406, "loss/value_avg": 0.61120594, "val/clipfrac_avg": 0.15411992, "policy/entropy_avg": 0.17828107, "val/ratio": 0.98907983, "val/ratio_var": 9.126e-05, "val/num_eos_tokens": 67, "lr": 9.74e-06, "episode": 2944, "epoch": 0.14868687, "global_step/max_steps": "184/1238", "percentage": "14.86%", "elapsed_time": "2h 4m 42s", "remaining_time": "11h 54m 22s"} +{"eps": 0, "objective/kl": 12.35282707, "objective/entropy": -101.67889404, "objective/non_score_reward": -0.61764133, "objective/rlhf_reward": 9.17923355, "objective/scores": 9.8125, "policy/approxkl_avg": 0.01111717, "policy/clipfrac_avg": 0.0173208, "loss/policy_avg": -0.18067747, "loss/value_avg": 0.52588773, "val/clipfrac_avg": 0.19647086, "policy/entropy_avg": 0.15565109, "val/ratio": 0.98465073, "val/ratio_var": 0.00082244, "val/num_eos_tokens": 210, "lr": 9.74e-06, "episode": 2960, "epoch": 0.14949495, "global_step/max_steps": "185/1238", "percentage": "14.94%", "elapsed_time": "2h 5m 35s", "remaining_time": "11h 54m 50s"} +{"eps": 0, "objective/kl": 11.29255676, "objective/entropy": -65.32836914, "objective/non_score_reward": -0.56462783, "objective/rlhf_reward": 9.22834015, "objective/scores": 9.8125, "policy/approxkl_avg": 0.00064677, "policy/clipfrac_avg": 0.02141393, "loss/policy_avg": 0.1885536, "loss/value_avg": 1.61094165, "val/clipfrac_avg": 0.18745771, "policy/entropy_avg": 0.11110902, "val/ratio": 0.99301624, "val/ratio_var": 3.98e-05, "val/num_eos_tokens": 246, "lr": 9.73e-06, "episode": 2976, "epoch": 0.15030303, "global_step/max_steps": "186/1238", "percentage": "15.02%", "elapsed_time": "2h 6m 16s", "remaining_time": "11h 54m 9s"} +{"eps": 0, "objective/kl": 16.00774765, "objective/entropy": -124.5298233, "objective/non_score_reward": -0.80038738, "objective/rlhf_reward": 7.24258184, "objective/scores": 8.0625, "policy/approxkl_avg": 0.00179357, "policy/clipfrac_avg": 0.02175932, "loss/policy_avg": 0.32616979, "loss/value_avg": 0.66815597, "val/clipfrac_avg": 0.12101784, "policy/entropy_avg": 0.15468502, "val/ratio": 0.98986471, "val/ratio_var": 5.843e-05, "val/num_eos_tokens": 21, "lr": 9.73e-06, "episode": 2992, "epoch": 0.15111111, "global_step/max_steps": "187/1238", "percentage": "15.11%", "elapsed_time": "2h 6m 51s", "remaining_time": "11h 52m 59s"} +{"eps": 0, "objective/kl": 14.36637878, "objective/entropy": -38.70436859, "objective/non_score_reward": -0.718319, "objective/rlhf_reward": 7.74652481, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00149091, "policy/clipfrac_avg": 0.02537131, "loss/policy_avg": 0.17494543, "loss/value_avg": 1.07903755, "val/clipfrac_avg": 0.11475747, "policy/entropy_avg": 0.15520191, "val/ratio": 0.98989046, "val/ratio_var": 7.074e-05, "val/num_eos_tokens": 135, "lr": 9.72e-06, "episode": 3008, "epoch": 0.15191919, "global_step/max_steps": "188/1238", "percentage": "15.19%", "elapsed_time": "2h 7m 35s", "remaining_time": "11h 52m 37s"} +{"eps": 0, "objective/kl": 13.32999134, "objective/entropy": -104.98526001, "objective/non_score_reward": -0.66649961, "objective/rlhf_reward": 7.05225039, "objective/scores": 7.71875, "policy/approxkl_avg": 0.00097027, "policy/clipfrac_avg": 0.03182618, "loss/policy_avg": -0.58332312, "loss/value_avg": 3.6844418, "val/clipfrac_avg": 0.15363654, "policy/entropy_avg": 0.17090809, "val/ratio": 0.98986661, "val/ratio_var": 0.00010602, "val/num_eos_tokens": 1103, "lr": 9.72e-06, "episode": 3024, "epoch": 0.15272727, "global_step/max_steps": "189/1238", "percentage": "15.27%", "elapsed_time": "2h 8m 55s", "remaining_time": "11h 55m 33s"} +{"eps": 0, "objective/kl": 11.61368656, "objective/entropy": -87.40807343, "objective/non_score_reward": -0.5806843, "objective/rlhf_reward": 10.41540909, "objective/scores": 11.0, "policy/approxkl_avg": 0.00099509, "policy/clipfrac_avg": 0.01073138, "loss/policy_avg": -0.14998452, "loss/value_avg": 2.0811553, "val/clipfrac_avg": 0.2018014, "policy/entropy_avg": 0.19691563, "val/ratio": 0.98875338, "val/ratio_var": 6.801e-05, "val/num_eos_tokens": 140, "lr": 9.71e-06, "episode": 3040, "epoch": 0.15353535, "global_step/max_steps": "190/1238", "percentage": "15.35%", "elapsed_time": "2h 9m 34s", "remaining_time": "11h 54m 44s"} +{"eps": 0, "objective/kl": 11.94738865, "objective/entropy": -76.6943512, "objective/non_score_reward": -0.59736943, "objective/rlhf_reward": 9.89481735, "objective/scores": 10.5, "policy/approxkl_avg": 0.00091299, "policy/clipfrac_avg": 0.00656932, "loss/policy_avg": -0.24951935, "loss/value_avg": 0.50001991, "val/clipfrac_avg": 0.13341542, "policy/entropy_avg": 0.16202569, "val/ratio": 0.98874474, "val/ratio_var": 5.932e-05, "val/num_eos_tokens": 435, "lr": 9.71e-06, "episode": 3056, "epoch": 0.15434343, "global_step/max_steps": "191/1238", "percentage": "15.43%", "elapsed_time": "2h 10m 9s", "remaining_time": "11h 53m 26s"} +{"eps": 0, "objective/kl": 13.22661304, "objective/entropy": -67.43034363, "objective/non_score_reward": -0.66133064, "objective/rlhf_reward": 10.83866882, "objective/scores": 11.5, "policy/approxkl_avg": 0.00125217, "policy/clipfrac_avg": 0.01195993, "loss/policy_avg": 0.07577252, "loss/value_avg": 0.5789153, "val/clipfrac_avg": 0.12880576, "policy/entropy_avg": 0.16521144, "val/ratio": 0.98819631, "val/ratio_var": 7.65e-05, "val/num_eos_tokens": 212, "lr": 9.71e-06, "episode": 3072, "epoch": 0.15515152, "global_step/max_steps": "192/1238", "percentage": "15.51%", "elapsed_time": "2h 10m 46s", "remaining_time": "11h 52m 26s"} +{"eps": 0, "objective/kl": 10.53251076, "objective/entropy": -95.40163422, "objective/non_score_reward": -0.52662563, "objective/rlhf_reward": 11.56321812, "objective/scores": 12.0625, "policy/approxkl_avg": 0.00097331, "policy/clipfrac_avg": 0.01573047, "loss/policy_avg": -0.41628423, "loss/value_avg": 2.15756774, "val/clipfrac_avg": 0.23112923, "policy/entropy_avg": 0.16752052, "val/ratio": 0.98864394, "val/ratio_var": 0.00013025, "val/num_eos_tokens": 344, "lr": 9.7e-06, "episode": 3088, "epoch": 0.1559596, "global_step/max_steps": "193/1238", "percentage": "15.59%", "elapsed_time": "2h 11m 23s", "remaining_time": "11h 51m 26s"} +{"eps": 0, "objective/kl": 11.38726044, "objective/entropy": -155.12109375, "objective/non_score_reward": -0.569363, "objective/rlhf_reward": 8.19040298, "objective/scores": 8.75, "policy/approxkl_avg": 0.00079244, "policy/clipfrac_avg": 0.01498166, "loss/policy_avg": 0.65622985, "loss/value_avg": 1.85428274, "val/clipfrac_avg": 0.21850237, "policy/entropy_avg": 0.15529108, "val/ratio": 0.9898361, "val/ratio_var": 0.00011535, "val/num_eos_tokens": 38, "lr": 9.7e-06, "episode": 3104, "epoch": 0.15676768, "global_step/max_steps": "194/1238", "percentage": "15.67%", "elapsed_time": "2h 12m 23s", "remaining_time": "11h 52m 26s"} +{"eps": 0, "objective/kl": 14.46889305, "objective/entropy": -108.78264618, "objective/non_score_reward": -0.7234447, "objective/rlhf_reward": 10.98749256, "objective/scores": 11.6875, "policy/approxkl_avg": 0.00115928, "policy/clipfrac_avg": 0.01232389, "loss/policy_avg": -0.2336686, "loss/value_avg": 1.75621665, "val/clipfrac_avg": 0.30650499, "policy/entropy_avg": 0.13213587, "val/ratio": 0.99179971, "val/ratio_var": 5.691e-05, "val/num_eos_tokens": 191, "lr": 9.69e-06, "episode": 3120, "epoch": 0.15757576, "global_step/max_steps": "195/1238", "percentage": "15.75%", "elapsed_time": "2h 13m 11s", "remaining_time": "11h 52m 24s"} +{"eps": 0, "objective/kl": 13.23812866, "objective/entropy": -72.25318909, "objective/non_score_reward": -0.66190648, "objective/rlhf_reward": 8.7755928, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00086796, "policy/clipfrac_avg": 0.0103099, "loss/policy_avg": -0.12489986, "loss/value_avg": 0.34016442, "val/clipfrac_avg": 0.11943725, "policy/entropy_avg": 0.16907692, "val/ratio": 0.99049866, "val/ratio_var": 6.889e-05, "val/num_eos_tokens": 279, "lr": 9.69e-06, "episode": 3136, "epoch": 0.15838384, "global_step/max_steps": "196/1238", "percentage": "15.83%", "elapsed_time": "2h 13m 39s", "remaining_time": "11h 50m 34s"} +{"eps": 0, "objective/kl": 15.96037388, "objective/entropy": -102.12807465, "objective/non_score_reward": -0.79801869, "objective/rlhf_reward": 9.14338684, "objective/scores": 9.9375, "policy/approxkl_avg": 0.00093817, "policy/clipfrac_avg": 0.01555368, "loss/policy_avg": -0.62712634, "loss/value_avg": 1.24002492, "val/clipfrac_avg": 0.17646958, "policy/entropy_avg": 0.17541647, "val/ratio": 0.98858356, "val/ratio_var": 5.869e-05, "val/num_eos_tokens": 110, "lr": 9.68e-06, "episode": 3152, "epoch": 0.15919192, "global_step/max_steps": "197/1238", "percentage": "15.91%", "elapsed_time": "2h 14m 22s", "remaining_time": "11h 50m 6s"} +{"eps": 0, "objective/kl": 11.87062931, "objective/entropy": -89.71901703, "objective/non_score_reward": -0.59353149, "objective/rlhf_reward": 9.84396839, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00088859, "policy/clipfrac_avg": 0.02810758, "loss/policy_avg": -0.03862653, "loss/value_avg": 0.63327312, "val/clipfrac_avg": 0.2914457, "policy/entropy_avg": 0.11816669, "val/ratio": 0.99145854, "val/ratio_var": 6.83e-05, "val/num_eos_tokens": 283, "lr": 9.68e-06, "episode": 3168, "epoch": 0.16, "global_step/max_steps": "198/1238", "percentage": "15.99%", "elapsed_time": "2h 14m 54s", "remaining_time": "11h 48m 34s"} +{"eps": 0, "objective/kl": 11.32013893, "objective/entropy": -124.32833862, "objective/non_score_reward": -0.5660069, "objective/rlhf_reward": 8.31680489, "objective/scores": 8.875, "policy/approxkl_avg": 0.00067076, "policy/clipfrac_avg": 0.0090035, "loss/policy_avg": 0.54308408, "loss/value_avg": 1.14722919, "val/clipfrac_avg": 0.1790837, "policy/entropy_avg": 0.19544983, "val/ratio": 0.99174297, "val/ratio_var": 4.731e-05, "val/num_eos_tokens": 554, "lr": 9.67e-06, "episode": 3184, "epoch": 0.16080808, "global_step/max_steps": "199/1238", "percentage": "16.07%", "elapsed_time": "2h 15m 24s", "remaining_time": "11h 46m 58s"} +{"eps": 0, "objective/kl": 12.47595215, "objective/entropy": -106.52888489, "objective/non_score_reward": -0.6237976, "objective/rlhf_reward": 7.98167133, "objective/scores": 8.625, "policy/approxkl_avg": 0.00098054, "policy/clipfrac_avg": 0.01122211, "loss/policy_avg": -0.02854505, "loss/value_avg": 0.87218142, "val/clipfrac_avg": 0.11438994, "policy/entropy_avg": 0.15164232, "val/ratio": 0.98802298, "val/ratio_var": 0.00015194, "val/num_eos_tokens": 13, "lr": 9.67e-06, "episode": 3200, "epoch": 0.16161616, "global_step/max_steps": "200/1238", "percentage": "16.16%", "elapsed_time": "2h 16m 1s", "remaining_time": "11h 45m 55s"} +{"eps": 0, "objective/kl": 10.96439934, "objective/entropy": -55.07182693, "objective/non_score_reward": -0.54821992, "objective/rlhf_reward": 8.70959282, "objective/scores": 9.25, "policy/approxkl_avg": 0.00276615, "policy/clipfrac_avg": 0.02268021, "loss/policy_avg": 0.03965133, "loss/value_avg": 0.8802923, "val/clipfrac_avg": 0.24667433, "policy/entropy_avg": 0.15708685, "val/ratio": 0.98827469, "val/ratio_var": 0.00016298, "val/num_eos_tokens": 274, "lr": 9.66e-06, "episode": 3216, "epoch": 0.16242424, "global_step/max_steps": "201/1238", "percentage": "16.24%", "elapsed_time": "2h 16m 54s", "remaining_time": "11h 46m 19s"} +{"eps": 0, "objective/kl": 12.81169891, "objective/entropy": -98.02118683, "objective/non_score_reward": -0.64058495, "objective/rlhf_reward": 7.71879005, "objective/scores": 8.375, "policy/approxkl_avg": 0.00107256, "policy/clipfrac_avg": 0.01077085, "loss/policy_avg": 0.31882551, "loss/value_avg": 1.16263115, "val/clipfrac_avg": 0.14545709, "policy/entropy_avg": 0.19526285, "val/ratio": 0.98718607, "val/ratio_var": 0.00014847, "val/num_eos_tokens": 143, "lr": 9.66e-06, "episode": 3232, "epoch": 0.16323232, "global_step/max_steps": "202/1238", "percentage": "16.32%", "elapsed_time": "2h 17m 52s", "remaining_time": "11h 47m 4s"} +{"eps": 0, "objective/kl": 16.77866364, "objective/entropy": -72.79200745, "objective/non_score_reward": -0.83893317, "objective/rlhf_reward": 7.25872326, "objective/scores": 8.125, "policy/approxkl_avg": 0.00095372, "policy/clipfrac_avg": 0.00741408, "loss/policy_avg": -0.03615336, "loss/value_avg": 0.21554255, "val/clipfrac_avg": 0.0817599, "policy/entropy_avg": 0.17590141, "val/ratio": 0.98854721, "val/ratio_var": 2.717e-05, "val/num_eos_tokens": 82, "lr": 9.65e-06, "episode": 3248, "epoch": 0.1640404, "global_step/max_steps": "203/1238", "percentage": "16.40%", "elapsed_time": "2h 18m 23s", "remaining_time": "11h 45m 35s"} +{"eps": 0, "objective/kl": 14.44842148, "objective/entropy": -123.76312256, "objective/non_score_reward": -0.72242105, "objective/rlhf_reward": 9.5080471, "objective/scores": 10.25, "policy/approxkl_avg": 0.00079302, "policy/clipfrac_avg": 0.01099657, "loss/policy_avg": -0.25860804, "loss/value_avg": 1.39656436, "val/clipfrac_avg": 0.13870262, "policy/entropy_avg": 0.14073229, "val/ratio": 0.99132425, "val/ratio_var": 6.24e-05, "val/num_eos_tokens": 154, "lr": 9.65e-06, "episode": 3264, "epoch": 0.16484848, "global_step/max_steps": "204/1238", "percentage": "16.48%", "elapsed_time": "2h 19m 10s", "remaining_time": "11h 45m 27s"} +{"eps": 0, "objective/kl": 11.82594872, "objective/entropy": -65.26087952, "objective/non_score_reward": -0.59129745, "objective/rlhf_reward": 9.1782341, "objective/scores": 9.75, "policy/approxkl_avg": 0.00078019, "policy/clipfrac_avg": 0.00578593, "loss/policy_avg": -0.58719802, "loss/value_avg": 1.17430425, "val/clipfrac_avg": 0.25026363, "policy/entropy_avg": 0.14145887, "val/ratio": 0.99062961, "val/ratio_var": 6.659e-05, "val/num_eos_tokens": 16, "lr": 9.64e-06, "episode": 3280, "epoch": 0.16565657, "global_step/max_steps": "205/1238", "percentage": "16.56%", "elapsed_time": "2h 19m 44s", "remaining_time": "11h 44m 9s"} +{"eps": 0, "objective/kl": 14.5935812, "objective/entropy": -76.86801147, "objective/non_score_reward": -0.72967905, "objective/rlhf_reward": 6.56719589, "objective/scores": 7.3125, "policy/approxkl_avg": 0.00087516, "policy/clipfrac_avg": 0.00959798, "loss/policy_avg": 0.2609019, "loss/value_avg": 0.48005229, "val/clipfrac_avg": 0.27932572, "policy/entropy_avg": 0.16647959, "val/ratio": 0.9888376, "val/ratio_var": 7.126e-05, "val/num_eos_tokens": 509, "lr": 9.64e-06, "episode": 3296, "epoch": 0.16646465, "global_step/max_steps": "206/1238", "percentage": "16.64%", "elapsed_time": "2h 20m 23s", "remaining_time": "11h 43m 17s"} +{"eps": 0, "objective/kl": 9.69997406, "objective/entropy": -58.05167389, "objective/non_score_reward": -0.48499873, "objective/rlhf_reward": 8.22984505, "objective/scores": 8.6875, "policy/approxkl_avg": 0.00085044, "policy/clipfrac_avg": 0.00808516, "loss/policy_avg": 0.10253682, "loss/value_avg": 1.08205843, "val/clipfrac_avg": 0.26196188, "policy/entropy_avg": 0.16680527, "val/ratio": 0.9909215, "val/ratio_var": 6.328e-05, "val/num_eos_tokens": 61, "lr": 9.63e-06, "episode": 3312, "epoch": 0.16727273, "global_step/max_steps": "207/1238", "percentage": "16.72%", "elapsed_time": "2h 20m 54s", "remaining_time": "11h 41m 50s"} +{"eps": 0, "objective/kl": 14.65669632, "objective/entropy": -32.68846512, "objective/non_score_reward": -0.73283482, "objective/rlhf_reward": 9.11482143, "objective/scores": 9.875, "policy/approxkl_avg": 0.00149946, "policy/clipfrac_avg": 0.012034, "loss/policy_avg": 0.12080384, "loss/value_avg": 0.77590549, "val/clipfrac_avg": 0.18890069, "policy/entropy_avg": 0.22088957, "val/ratio": 0.98681527, "val/ratio_var": 0.00015818, "val/num_eos_tokens": 364, "lr": 9.63e-06, "episode": 3328, "epoch": 0.16808081, "global_step/max_steps": "208/1238", "percentage": "16.80%", "elapsed_time": "2h 21m 34s", "remaining_time": "11h 41m 6s"} +{"eps": 0, "objective/kl": 15.4480381, "objective/entropy": -99.28005219, "objective/non_score_reward": -0.77240193, "objective/rlhf_reward": 7.68853569, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00084678, "policy/clipfrac_avg": 0.00758356, "loss/policy_avg": 0.01144018, "loss/value_avg": 0.22326101, "val/clipfrac_avg": 0.05371936, "policy/entropy_avg": 0.15184712, "val/ratio": 0.98953426, "val/ratio_var": 4.037e-05, "val/num_eos_tokens": 255, "lr": 9.62e-06, "episode": 3344, "epoch": 0.16888889, "global_step/max_steps": "209/1238", "percentage": "16.88%", "elapsed_time": "2h 22m 22s", "remaining_time": "11h 40m 57s"} +{"eps": 0, "objective/kl": 13.33155823, "objective/entropy": -74.21755219, "objective/non_score_reward": -0.66657794, "objective/rlhf_reward": 7.83342218, "objective/scores": 8.5, "policy/approxkl_avg": 0.00100895, "policy/clipfrac_avg": 0.00991973, "loss/policy_avg": 0.0517005, "loss/value_avg": 0.70919353, "val/clipfrac_avg": 0.29762989, "policy/entropy_avg": 0.19567251, "val/ratio": 0.9870832, "val/ratio_var": 6.727e-05, "val/num_eos_tokens": 599, "lr": 9.62e-06, "episode": 3360, "epoch": 0.16969697, "global_step/max_steps": "210/1238", "percentage": "16.96%", "elapsed_time": "2h 23m 0s", "remaining_time": "11h 40m 1s"} +{"eps": 0, "objective/kl": 13.6245985, "objective/entropy": -130.67494202, "objective/non_score_reward": -0.68122995, "objective/rlhf_reward": 9.25627041, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0007515, "policy/clipfrac_avg": 0.00542304, "loss/policy_avg": -0.44115239, "loss/value_avg": 0.55157328, "val/clipfrac_avg": 0.29410303, "policy/entropy_avg": 0.15925181, "val/ratio": 0.99012846, "val/ratio_var": 5.974e-05, "val/num_eos_tokens": 550, "lr": 9.61e-06, "episode": 3376, "epoch": 0.17050505, "global_step/max_steps": "211/1238", "percentage": "17.04%", "elapsed_time": "2h 23m 41s", "remaining_time": "11h 39m 25s"} +{"eps": 0, "objective/kl": 13.88901901, "objective/entropy": -25.59563446, "objective/non_score_reward": -0.69445097, "objective/rlhf_reward": 7.41101742, "objective/scores": 8.125, "policy/approxkl_avg": 0.00095751, "policy/clipfrac_avg": 0.00908328, "loss/policy_avg": 0.00408672, "loss/value_avg": 0.23123895, "val/clipfrac_avg": 0.17202628, "policy/entropy_avg": 0.15931988, "val/ratio": 0.98936617, "val/ratio_var": 3.22e-05, "val/num_eos_tokens": 129, "lr": 9.61e-06, "episode": 3392, "epoch": 0.17131313, "global_step/max_steps": "212/1238", "percentage": "17.12%", "elapsed_time": "2h 24m 19s", "remaining_time": "11h 38m 27s"} +{"eps": 0, "objective/kl": 16.51137543, "objective/entropy": -68.18078613, "objective/non_score_reward": -0.82556874, "objective/rlhf_reward": 7.79161835, "objective/scores": 8.625, "policy/approxkl_avg": 0.00098548, "policy/clipfrac_avg": 0.00542449, "loss/policy_avg": 0.02707344, "loss/value_avg": 0.22074272, "val/clipfrac_avg": 0.06424005, "policy/entropy_avg": 0.17797673, "val/ratio": 0.98748004, "val/ratio_var": 5.017e-05, "val/num_eos_tokens": 11, "lr": 9.6e-06, "episode": 3408, "epoch": 0.17212121, "global_step/max_steps": "213/1238", "percentage": "17.21%", "elapsed_time": "2h 24m 59s", "remaining_time": "11h 37m 44s"} +{"eps": 0, "objective/kl": 14.35812855, "objective/entropy": -147.71899414, "objective/non_score_reward": -0.71790648, "objective/rlhf_reward": 9.0125618, "objective/scores": 9.75, "policy/approxkl_avg": 0.00089983, "policy/clipfrac_avg": 0.00601834, "loss/policy_avg": -0.22705355, "loss/value_avg": 0.47900987, "val/clipfrac_avg": 0.08306095, "policy/entropy_avg": 0.16658711, "val/ratio": 0.98892832, "val/ratio_var": 7.234e-05, "val/num_eos_tokens": 616, "lr": 9.6e-06, "episode": 3424, "epoch": 0.17292929, "global_step/max_steps": "214/1238", "percentage": "17.29%", "elapsed_time": "2h 25m 39s", "remaining_time": "11h 36m 59s"} +{"eps": 0, "objective/kl": 9.83112717, "objective/entropy": -91.03656769, "objective/non_score_reward": -0.49155638, "objective/rlhf_reward": 9.99281883, "objective/scores": 10.5, "policy/approxkl_avg": 0.00107484, "policy/clipfrac_avg": 0.02250077, "loss/policy_avg": -0.1495879, "loss/value_avg": 2.32874155, "val/clipfrac_avg": 0.27161676, "policy/entropy_avg": 0.19421768, "val/ratio": 0.98804677, "val/ratio_var": 0.00019949, "val/num_eos_tokens": 249, "lr": 9.59e-06, "episode": 3440, "epoch": 0.17373737, "global_step/max_steps": "215/1238", "percentage": "17.37%", "elapsed_time": "2h 26m 14s", "remaining_time": "11h 35m 48s"} +{"eps": 0, "objective/kl": 11.4323864, "objective/entropy": -65.73722839, "objective/non_score_reward": -0.57161927, "objective/rlhf_reward": 9.63541222, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00087288, "policy/clipfrac_avg": 0.01602334, "loss/policy_avg": 0.25549322, "loss/value_avg": 1.37580621, "val/clipfrac_avg": 0.1290468, "policy/entropy_avg": 0.16358376, "val/ratio": 0.99076915, "val/ratio_var": 4.8e-05, "val/num_eos_tokens": 12, "lr": 9.59e-06, "episode": 3456, "epoch": 0.17454545, "global_step/max_steps": "216/1238", "percentage": "17.45%", "elapsed_time": "2h 26m 50s", "remaining_time": "11h 34m 47s"} +{"eps": 0, "objective/kl": 13.61048889, "objective/entropy": -72.13745117, "objective/non_score_reward": -0.68052447, "objective/rlhf_reward": 8.85853767, "objective/scores": 9.5625, "policy/approxkl_avg": 0.00093415, "policy/clipfrac_avg": 0.00695694, "loss/policy_avg": 0.39980143, "loss/value_avg": 0.74311948, "val/clipfrac_avg": 0.10816794, "policy/entropy_avg": 0.18271828, "val/ratio": 0.99068385, "val/ratio_var": 9.381e-05, "val/num_eos_tokens": 17, "lr": 9.58e-06, "episode": 3472, "epoch": 0.17535354, "global_step/max_steps": "217/1238", "percentage": "17.53%", "elapsed_time": "2h 27m 33s", "remaining_time": "11h 34m 16s"} +{"eps": 0, "objective/kl": 14.33843994, "objective/entropy": -89.79592896, "objective/non_score_reward": -0.71692199, "objective/rlhf_reward": 9.43932724, "objective/scores": 10.125, "policy/approxkl_avg": 0.00100627, "policy/clipfrac_avg": 0.01650636, "loss/policy_avg": 0.25006008, "loss/value_avg": 0.91197485, "val/clipfrac_avg": 0.07429616, "policy/entropy_avg": 0.17433929, "val/ratio": 0.98911643, "val/ratio_var": 4.176e-05, "val/num_eos_tokens": 11, "lr": 9.58e-06, "episode": 3488, "epoch": 0.17616162, "global_step/max_steps": "218/1238", "percentage": "17.61%", "elapsed_time": "2h 28m 14s", "remaining_time": "11h 33m 37s"} +{"eps": 0, "objective/kl": 12.96816063, "objective/entropy": -125.2816925, "objective/non_score_reward": -0.64840806, "objective/rlhf_reward": 7.79299831, "objective/scores": 8.4375, "policy/approxkl_avg": 0.00072754, "policy/clipfrac_avg": 0.00535131, "loss/policy_avg": -0.24181823, "loss/value_avg": 0.22863328, "val/clipfrac_avg": 0.17199305, "policy/entropy_avg": 0.14211547, "val/ratio": 0.99092555, "val/ratio_var": 6.175e-05, "val/num_eos_tokens": 123, "lr": 9.57e-06, "episode": 3504, "epoch": 0.1769697, "global_step/max_steps": "219/1238", "percentage": "17.69%", "elapsed_time": "2h 28m 48s", "remaining_time": "11h 32m 23s"} +{"eps": 0, "objective/kl": 9.07908344, "objective/entropy": -49.96704102, "objective/non_score_reward": -0.45395419, "objective/rlhf_reward": 10.4522953, "objective/scores": 10.875, "policy/approxkl_avg": 0.00087023, "policy/clipfrac_avg": 0.00654862, "loss/policy_avg": -0.12509485, "loss/value_avg": 0.60275602, "val/clipfrac_avg": 0.08971221, "policy/entropy_avg": 0.16074753, "val/ratio": 0.99017823, "val/ratio_var": 4.497e-05, "val/num_eos_tokens": 193, "lr": 9.57e-06, "episode": 3520, "epoch": 0.17777778, "global_step/max_steps": "220/1238", "percentage": "17.77%", "elapsed_time": "2h 29m 21s", "remaining_time": "11h 31m 5s"} +{"eps": 0, "objective/kl": 15.14925385, "objective/entropy": -29.95349312, "objective/non_score_reward": -0.75746274, "objective/rlhf_reward": 8.06285, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00100213, "policy/clipfrac_avg": 0.00552067, "loss/policy_avg": -0.00034147, "loss/value_avg": 0.40313023, "val/clipfrac_avg": 0.09643255, "policy/entropy_avg": 0.19618559, "val/ratio": 0.98748744, "val/ratio_var": 3.069e-05, "val/num_eos_tokens": 200, "lr": 9.56e-06, "episode": 3536, "epoch": 0.17858586, "global_step/max_steps": "221/1238", "percentage": "17.85%", "elapsed_time": "2h 30m 7s", "remaining_time": "11h 30m 52s"} +{"eps": 0, "objective/kl": 9.77555466, "objective/entropy": -90.66460419, "objective/non_score_reward": -0.48877776, "objective/rlhf_reward": 11.56591034, "objective/scores": 12.0625, "policy/approxkl_avg": 0.00099398, "policy/clipfrac_avg": 0.00786105, "loss/policy_avg": -0.0959366, "loss/value_avg": 2.48357773, "val/clipfrac_avg": 0.22381994, "policy/entropy_avg": 0.18746948, "val/ratio": 0.98904568, "val/ratio_var": 6.582e-05, "val/num_eos_tokens": 27, "lr": 9.56e-06, "episode": 3552, "epoch": 0.17939394, "global_step/max_steps": "222/1238", "percentage": "17.93%", "elapsed_time": "2h 30m 45s", "remaining_time": "11h 29m 57s"} +{"eps": 0, "objective/kl": 17.49753952, "objective/entropy": -154.86912537, "objective/non_score_reward": -0.87487698, "objective/rlhf_reward": 8.82434177, "objective/scores": 9.6875, "policy/approxkl_avg": 0.00086056, "policy/clipfrac_avg": 0.0057136, "loss/policy_avg": -0.23206055, "loss/value_avg": 0.20906797, "val/clipfrac_avg": 0.09188984, "policy/entropy_avg": 0.17575836, "val/ratio": 0.98838389, "val/ratio_var": 6.594e-05, "val/num_eos_tokens": 225, "lr": 9.55e-06, "episode": 3568, "epoch": 0.18020202, "global_step/max_steps": "223/1238", "percentage": "18.01%", "elapsed_time": "2h 32m 21s", "remaining_time": "11h 33m 30s"} +{"eps": 0, "objective/kl": 12.19713879, "objective/entropy": -123.72830963, "objective/non_score_reward": -0.60985696, "objective/rlhf_reward": 9.11670494, "objective/scores": 9.75, "policy/approxkl_avg": 0.0009077, "policy/clipfrac_avg": 0.02235701, "loss/policy_avg": -0.08118355, "loss/value_avg": 1.01160693, "val/clipfrac_avg": 0.12505849, "policy/entropy_avg": 0.19714952, "val/ratio": 0.98909754, "val/ratio_var": 5.287e-05, "val/num_eos_tokens": 310, "lr": 9.54e-06, "episode": 3584, "epoch": 0.1810101, "global_step/max_steps": "224/1238", "percentage": "18.09%", "elapsed_time": "2h 33m 18s", "remaining_time": "11h 33m 57s"} +{"eps": 0, "objective/kl": 13.42066193, "objective/entropy": -127.17562866, "objective/non_score_reward": -0.67103314, "objective/rlhf_reward": 8.62584209, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00078864, "policy/clipfrac_avg": 0.00675695, "loss/policy_avg": 0.03267436, "loss/value_avg": 1.11745894, "val/clipfrac_avg": 0.13264839, "policy/entropy_avg": 0.15594673, "val/ratio": 0.98962522, "val/ratio_var": 5.248e-05, "val/num_eos_tokens": 510, "lr": 9.54e-06, "episode": 3600, "epoch": 0.18181818, "global_step/max_steps": "225/1238", "percentage": "18.17%", "elapsed_time": "2h 33m 52s", "remaining_time": "11h 32m 45s"} +{"eps": 0, "objective/kl": 18.10074615, "objective/entropy": -67.1109314, "objective/non_score_reward": -0.90503734, "objective/rlhf_reward": 8.50902557, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00108458, "policy/clipfrac_avg": 0.00887345, "loss/policy_avg": -0.27726984, "loss/value_avg": 0.64624983, "val/clipfrac_avg": 0.0888463, "policy/entropy_avg": 0.17022252, "val/ratio": 0.98778272, "val/ratio_var": 4.778e-05, "val/num_eos_tokens": 123, "lr": 9.53e-06, "episode": 3616, "epoch": 0.18262626, "global_step/max_steps": "226/1238", "percentage": "18.26%", "elapsed_time": "2h 34m 34s", "remaining_time": "11h 32m 11s"} +{"eps": 0, "objective/kl": 15.78511715, "objective/entropy": -100.66576385, "objective/non_score_reward": -0.78925586, "objective/rlhf_reward": 8.71074486, "objective/scores": 9.5, "policy/approxkl_avg": 0.00106304, "policy/clipfrac_avg": 0.01022335, "loss/policy_avg": 0.15851098, "loss/value_avg": 0.51683199, "val/clipfrac_avg": 0.06462044, "policy/entropy_avg": 0.19960141, "val/ratio": 0.98717344, "val/ratio_var": 6.318e-05, "val/num_eos_tokens": 393, "lr": 9.53e-06, "episode": 3632, "epoch": 0.18343434, "global_step/max_steps": "227/1238", "percentage": "18.34%", "elapsed_time": "2h 35m 11s", "remaining_time": "11h 31m 12s"} +{"eps": 0, "objective/kl": 12.63337135, "objective/entropy": -114.29866791, "objective/non_score_reward": -0.63166857, "objective/rlhf_reward": 8.78630066, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00083456, "policy/clipfrac_avg": 0.00470011, "loss/policy_avg": -0.31773216, "loss/value_avg": 0.84598196, "val/clipfrac_avg": 0.1266717, "policy/entropy_avg": 0.16971874, "val/ratio": 0.98971713, "val/ratio_var": 5.563e-05, "val/num_eos_tokens": 116, "lr": 9.52e-06, "episode": 3648, "epoch": 0.18424242, "global_step/max_steps": "228/1238", "percentage": "18.42%", "elapsed_time": "2h 35m 50s", "remaining_time": "11h 30m 19s"} +{"eps": 0, "objective/kl": 12.76868248, "objective/entropy": -71.81008148, "objective/non_score_reward": -0.63843411, "objective/rlhf_reward": 8.95531559, "objective/scores": 9.625, "policy/approxkl_avg": 0.00089874, "policy/clipfrac_avg": 0.01049551, "loss/policy_avg": 0.04545651, "loss/value_avg": 1.42191029, "val/clipfrac_avg": 0.17400485, "policy/entropy_avg": 0.17481613, "val/ratio": 0.98926765, "val/ratio_var": 7.377e-05, "val/num_eos_tokens": 326, "lr": 9.52e-06, "episode": 3664, "epoch": 0.18505051, "global_step/max_steps": "229/1238", "percentage": "18.50%", "elapsed_time": "2h 36m 22s", "remaining_time": "11h 29m 0s"} +{"eps": 0, "objective/kl": 13.7195425, "objective/entropy": -140.62866211, "objective/non_score_reward": -0.6859771, "objective/rlhf_reward": 8.63042927, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00077484, "policy/clipfrac_avg": 0.00818425, "loss/policy_avg": 0.03276506, "loss/value_avg": 0.51399118, "val/clipfrac_avg": 0.15990712, "policy/entropy_avg": 0.15187263, "val/ratio": 0.99081993, "val/ratio_var": 4.362e-05, "val/num_eos_tokens": 475, "lr": 9.51e-06, "episode": 3680, "epoch": 0.18585859, "global_step/max_steps": "230/1238", "percentage": "18.58%", "elapsed_time": "2h 37m 3s", "remaining_time": "11h 28m 21s"} +{"eps": 0, "objective/kl": 13.59559631, "objective/entropy": -112.58895874, "objective/non_score_reward": -0.67977989, "objective/rlhf_reward": 6.99990749, "objective/scores": 7.6875, "policy/approxkl_avg": 0.00079511, "policy/clipfrac_avg": 0.01423525, "loss/policy_avg": 0.35437015, "loss/value_avg": 0.72937429, "val/clipfrac_avg": 0.1404144, "policy/entropy_avg": 0.13730276, "val/ratio": 0.99055314, "val/ratio_var": 4.273e-05, "val/num_eos_tokens": 31, "lr": 9.5e-06, "episode": 3696, "epoch": 0.18666667, "global_step/max_steps": "231/1238", "percentage": "18.66%", "elapsed_time": "2h 37m 38s", "remaining_time": "11h 27m 10s"} +{"eps": 0, "objective/kl": 15.55249023, "objective/entropy": -95.20710754, "objective/non_score_reward": -0.77762449, "objective/rlhf_reward": 8.19503212, "objective/scores": 9.0, "policy/approxkl_avg": 0.00092951, "policy/clipfrac_avg": 0.00644633, "loss/policy_avg": 0.07381056, "loss/value_avg": 0.37465847, "val/clipfrac_avg": 0.08955052, "policy/entropy_avg": 0.17013073, "val/ratio": 0.98856884, "val/ratio_var": 4.925e-05, "val/num_eos_tokens": 338, "lr": 9.5e-06, "episode": 3712, "epoch": 0.18747475, "global_step/max_steps": "232/1238", "percentage": "18.74%", "elapsed_time": "2h 38m 16s", "remaining_time": "11h 26m 19s"} +{"eps": 0, "objective/kl": 11.42530251, "objective/entropy": -55.17114258, "objective/non_score_reward": -0.5712651, "objective/rlhf_reward": 11.71779728, "objective/scores": 12.3125, "policy/approxkl_avg": 0.00099249, "policy/clipfrac_avg": 0.01582335, "loss/policy_avg": -0.37431696, "loss/value_avg": 1.44899046, "val/clipfrac_avg": 0.34639126, "policy/entropy_avg": 0.15912247, "val/ratio": 0.99042761, "val/ratio_var": 4.265e-05, "val/num_eos_tokens": 212, "lr": 9.49e-06, "episode": 3728, "epoch": 0.18828283, "global_step/max_steps": "233/1238", "percentage": "18.82%", "elapsed_time": "2h 38m 51s", "remaining_time": "11h 25m 11s"} +{"eps": 0, "objective/kl": 11.21828079, "objective/entropy": -114.34797668, "objective/non_score_reward": -0.56091404, "objective/rlhf_reward": 6.79846096, "objective/scores": 7.375, "policy/approxkl_avg": 0.00138484, "policy/clipfrac_avg": 0.01319712, "loss/policy_avg": 0.12119548, "loss/value_avg": 1.03383923, "val/clipfrac_avg": 0.25744998, "policy/entropy_avg": 0.19113111, "val/ratio": 0.98918003, "val/ratio_var": 6.691e-05, "val/num_eos_tokens": 162, "lr": 9.49e-06, "episode": 3744, "epoch": 0.18909091, "global_step/max_steps": "234/1238", "percentage": "18.90%", "elapsed_time": "2h 39m 31s", "remaining_time": "11h 24m 28s"} +{"eps": 0, "objective/kl": 14.93826675, "objective/entropy": -180.02139282, "objective/non_score_reward": -0.74691331, "objective/rlhf_reward": 8.26871109, "objective/scores": 9.0, "policy/approxkl_avg": 0.00069309, "policy/clipfrac_avg": 0.01122877, "loss/policy_avg": -0.0545643, "loss/value_avg": 2.05128527, "val/clipfrac_avg": 0.1602141, "policy/entropy_avg": 0.1218586, "val/ratio": 0.99173033, "val/ratio_var": 3.994e-05, "val/num_eos_tokens": 303, "lr": 9.48e-06, "episode": 3760, "epoch": 0.18989899, "global_step/max_steps": "235/1238", "percentage": "18.98%", "elapsed_time": "2h 41m 18s", "remaining_time": "11h 28m 27s"} +{"eps": 0, "objective/kl": 14.79059982, "objective/entropy": -145.4604187, "objective/non_score_reward": -0.73952997, "objective/rlhf_reward": 10.03390694, "objective/scores": 10.75, "policy/approxkl_avg": 0.00087204, "policy/clipfrac_avg": 0.0056995, "loss/policy_avg": -0.27526289, "loss/value_avg": 0.68809199, "val/clipfrac_avg": 0.06145747, "policy/entropy_avg": 0.16308784, "val/ratio": 0.98930931, "val/ratio_var": 5.271e-05, "val/num_eos_tokens": 592, "lr": 9.48e-06, "episode": 3776, "epoch": 0.19070707, "global_step/max_steps": "236/1238", "percentage": "19.06%", "elapsed_time": "2h 42m 0s", "remaining_time": "11h 27m 50s"} +{"eps": 0, "objective/kl": 14.9547863, "objective/entropy": -107.19807434, "objective/non_score_reward": -0.74773932, "objective/rlhf_reward": 8.98272896, "objective/scores": 9.75, "policy/approxkl_avg": 0.00110509, "policy/clipfrac_avg": 0.03215779, "loss/policy_avg": 0.15451846, "loss/value_avg": 1.84764993, "val/clipfrac_avg": 0.08973571, "policy/entropy_avg": 0.18476415, "val/ratio": 0.98819566, "val/ratio_var": 5.365e-05, "val/num_eos_tokens": 680, "lr": 9.47e-06, "episode": 3792, "epoch": 0.19151515, "global_step/max_steps": "237/1238", "percentage": "19.14%", "elapsed_time": "2h 42m 43s", "remaining_time": "11h 27m 18s"} +{"eps": 0, "objective/kl": 15.00160885, "objective/entropy": -146.70474243, "objective/non_score_reward": -0.75008035, "objective/rlhf_reward": 7.75382566, "objective/scores": 8.5, "policy/approxkl_avg": 0.0008399, "policy/clipfrac_avg": 0.00597469, "loss/policy_avg": -0.00788788, "loss/value_avg": 0.22297966, "val/clipfrac_avg": 0.08070166, "policy/entropy_avg": 0.15901756, "val/ratio": 0.98944628, "val/ratio_var": 6.363e-05, "val/num_eos_tokens": 343, "lr": 9.46e-06, "episode": 3808, "epoch": 0.19232323, "global_step/max_steps": "238/1238", "percentage": "19.22%", "elapsed_time": "2h 43m 20s", "remaining_time": "11h 26m 19s"} +{"eps": 0, "objective/kl": 11.3671875, "objective/entropy": -49.81889343, "objective/non_score_reward": -0.56835938, "objective/rlhf_reward": 9.86914062, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00115852, "policy/clipfrac_avg": 0.0072056, "loss/policy_avg": 0.06090905, "loss/value_avg": 0.86475784, "val/clipfrac_avg": 0.13327752, "policy/entropy_avg": 0.18108273, "val/ratio": 0.98744231, "val/ratio_var": 7.535e-05, "val/num_eos_tokens": 15, "lr": 9.46e-06, "episode": 3824, "epoch": 0.19313131, "global_step/max_steps": "239/1238", "percentage": "19.31%", "elapsed_time": "2h 43m 54s", "remaining_time": "11h 25m 9s"} +{"eps": 0, "objective/kl": 13.6833744, "objective/entropy": -92.73252869, "objective/non_score_reward": -0.6841687, "objective/rlhf_reward": 9.84708118, "objective/scores": 10.5, "policy/approxkl_avg": 0.00098734, "policy/clipfrac_avg": 0.01126328, "loss/policy_avg": -0.00281322, "loss/value_avg": 0.47351715, "val/clipfrac_avg": 0.1254852, "policy/entropy_avg": 0.1566, "val/ratio": 0.99016649, "val/ratio_var": 5.156e-05, "val/num_eos_tokens": 335, "lr": 9.45e-06, "episode": 3840, "epoch": 0.19393939, "global_step/max_steps": "240/1238", "percentage": "19.39%", "elapsed_time": "2h 44m 39s", "remaining_time": "11h 24m 42s"} +{"eps": 0, "objective/kl": 14.89835167, "objective/entropy": -94.36567688, "objective/non_score_reward": -0.74491763, "objective/rlhf_reward": 9.73164463, "objective/scores": 10.5, "policy/approxkl_avg": 0.0008329, "policy/clipfrac_avg": 0.00558184, "loss/policy_avg": -0.11238708, "loss/value_avg": 0.30569232, "val/clipfrac_avg": 0.14762382, "policy/entropy_avg": 0.16306019, "val/ratio": 0.9892506, "val/ratio_var": 4.001e-05, "val/num_eos_tokens": 797, "lr": 9.45e-06, "episode": 3856, "epoch": 0.19474747, "global_step/max_steps": "241/1238", "percentage": "19.47%", "elapsed_time": "2h 45m 24s", "remaining_time": "11h 24m 17s"} +{"eps": 0, "objective/kl": 13.85297966, "objective/entropy": -108.59629822, "objective/non_score_reward": -0.69264901, "objective/rlhf_reward": 9.35422516, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00086564, "policy/clipfrac_avg": 0.00892029, "loss/policy_avg": 0.11825012, "loss/value_avg": 0.41407973, "val/clipfrac_avg": 0.23587155, "policy/entropy_avg": 0.16161919, "val/ratio": 0.98958731, "val/ratio_var": 5.987e-05, "val/num_eos_tokens": 95, "lr": 9.44e-06, "episode": 3872, "epoch": 0.19555556, "global_step/max_steps": "242/1238", "percentage": "19.55%", "elapsed_time": "2h 46m 6s", "remaining_time": "11h 23m 37s"} +{"eps": 0, "objective/kl": 20.37476349, "objective/entropy": -158.32139587, "objective/non_score_reward": -1.01873827, "objective/rlhf_reward": 7.76641846, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00084697, "policy/clipfrac_avg": 0.01724826, "loss/policy_avg": -0.12114518, "loss/value_avg": 0.56178951, "val/clipfrac_avg": 0.18548328, "policy/entropy_avg": 0.17039669, "val/ratio": 0.98961067, "val/ratio_var": 5.761e-05, "val/num_eos_tokens": 629, "lr": 9.43e-06, "episode": 3888, "epoch": 0.19636364, "global_step/max_steps": "243/1238", "percentage": "19.63%", "elapsed_time": "2h 46m 52s", "remaining_time": "11h 23m 19s"} +{"eps": 0, "objective/kl": 13.42576981, "objective/entropy": -76.74047852, "objective/non_score_reward": -0.67128849, "objective/rlhf_reward": 8.80527401, "objective/scores": 9.5, "policy/approxkl_avg": 0.00086735, "policy/clipfrac_avg": 0.01012692, "loss/policy_avg": 0.11465315, "loss/value_avg": 0.38835073, "val/clipfrac_avg": 0.08817488, "policy/entropy_avg": 0.14852989, "val/ratio": 0.98934209, "val/ratio_var": 4.652e-05, "val/num_eos_tokens": 105, "lr": 9.43e-06, "episode": 3904, "epoch": 0.19717172, "global_step/max_steps": "244/1238", "percentage": "19.71%", "elapsed_time": "2h 47m 23s", "remaining_time": "11h 21m 53s"} +{"eps": 0, "objective/kl": 14.99024868, "objective/entropy": -103.82775879, "objective/non_score_reward": -0.74951243, "objective/rlhf_reward": 7.5063467, "objective/scores": 8.25, "policy/approxkl_avg": 0.00087649, "policy/clipfrac_avg": 0.01041483, "loss/policy_avg": 0.07142085, "loss/value_avg": 0.46755302, "val/clipfrac_avg": 0.16199972, "policy/entropy_avg": 0.17772126, "val/ratio": 0.98905647, "val/ratio_var": 5.062e-05, "val/num_eos_tokens": 463, "lr": 9.42e-06, "episode": 3920, "epoch": 0.1979798, "global_step/max_steps": "245/1238", "percentage": "19.79%", "elapsed_time": "2h 48m 2s", "remaining_time": "11h 21m 5s"} +{"eps": 0, "objective/kl": 15.05043221, "objective/entropy": -68.2171402, "objective/non_score_reward": -0.75252163, "objective/rlhf_reward": 8.28654099, "objective/scores": 9.0625, "policy/approxkl_avg": 0.00132084, "policy/clipfrac_avg": 0.01767657, "loss/policy_avg": 0.09155969, "loss/value_avg": 1.2206161, "val/clipfrac_avg": 0.12387647, "policy/entropy_avg": 0.25825548, "val/ratio": 0.98643452, "val/ratio_var": 7.634e-05, "val/num_eos_tokens": 183, "lr": 9.41e-06, "episode": 3936, "epoch": 0.19878788, "global_step/max_steps": "246/1238", "percentage": "19.87%", "elapsed_time": "2h 48m 34s", "remaining_time": "11h 19m 48s"} +{"eps": 0, "objective/kl": 12.25468826, "objective/entropy": -98.11519623, "objective/non_score_reward": -0.61273444, "objective/rlhf_reward": 7.92632818, "objective/scores": 8.5625, "policy/approxkl_avg": 0.00067651, "policy/clipfrac_avg": 0.004696, "loss/policy_avg": 0.10812593, "loss/value_avg": 0.38382858, "val/clipfrac_avg": 0.27838826, "policy/entropy_avg": 0.13373852, "val/ratio": 0.99121451, "val/ratio_var": 3.24e-05, "val/num_eos_tokens": 74, "lr": 9.41e-06, "episode": 3952, "epoch": 0.19959596, "global_step/max_steps": "247/1238", "percentage": "19.95%", "elapsed_time": "2h 49m 8s", "remaining_time": "11h 18m 38s"} +{"eps": 0, "objective/kl": 14.07929325, "objective/entropy": -93.28247833, "objective/non_score_reward": -0.70396471, "objective/rlhf_reward": 8.09290981, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00101517, "policy/clipfrac_avg": 0.0107104, "loss/policy_avg": 0.25261652, "loss/value_avg": 0.49756327, "val/clipfrac_avg": 0.09553746, "policy/entropy_avg": 0.17333317, "val/ratio": 0.98876029, "val/ratio_var": 6.146e-05, "val/num_eos_tokens": 329, "lr": 9.4e-06, "episode": 3968, "epoch": 0.20040404, "global_step/max_steps": "248/1238", "percentage": "20.03%", "elapsed_time": "2h 50m 17s", "remaining_time": "11h 19m 47s"} +{"eps": 0, "objective/kl": 15.15790749, "objective/entropy": -106.61834717, "objective/non_score_reward": -0.75789535, "objective/rlhf_reward": 9.17179298, "objective/scores": 9.9375, "policy/approxkl_avg": 0.00089727, "policy/clipfrac_avg": 0.01750042, "loss/policy_avg": -0.33867991, "loss/value_avg": 0.64495695, "val/clipfrac_avg": 0.16914868, "policy/entropy_avg": 0.16986978, "val/ratio": 0.99003184, "val/ratio_var": 6.467e-05, "val/num_eos_tokens": 107, "lr": 9.4e-06, "episode": 3984, "epoch": 0.20121212, "global_step/max_steps": "249/1238", "percentage": "20.11%", "elapsed_time": "2h 50m 55s", "remaining_time": "11h 18m 55s"} +{"eps": 0, "objective/kl": 15.84797859, "objective/entropy": -88.12916565, "objective/non_score_reward": -0.79239887, "objective/rlhf_reward": 9.2818203, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00081809, "policy/clipfrac_avg": 0.00738172, "loss/policy_avg": -0.26445556, "loss/value_avg": 1.82425141, "val/clipfrac_avg": 0.09538545, "policy/entropy_avg": 0.15668297, "val/ratio": 0.98945236, "val/ratio_var": 4.987e-05, "val/num_eos_tokens": 460, "lr": 9.39e-06, "episode": 4000, "epoch": 0.2020202, "global_step/max_steps": "250/1238", "percentage": "20.19%", "elapsed_time": "2h 51m 31s", "remaining_time": "11h 17m 53s"} +{"eps": 0, "objective/kl": 15.98152637, "objective/entropy": -53.91804123, "objective/non_score_reward": -0.79907632, "objective/rlhf_reward": 7.15795469, "objective/scores": 7.96875, "policy/approxkl_avg": 0.00094866, "policy/clipfrac_avg": 0.00841789, "loss/policy_avg": -0.03636375, "loss/value_avg": 0.241541, "val/clipfrac_avg": 0.13178352, "policy/entropy_avg": 0.18229675, "val/ratio": 0.98804462, "val/ratio_var": 4.494e-05, "val/num_eos_tokens": 305, "lr": 9.38e-06, "episode": 4016, "epoch": 0.20282828, "global_step/max_steps": "251/1238", "percentage": "20.27%", "elapsed_time": "2h 52m 10s", "remaining_time": "11h 17m 1s"} +{"eps": 0, "objective/kl": 8.78524208, "objective/entropy": -116.62461853, "objective/non_score_reward": -0.43926215, "objective/rlhf_reward": 9.60370636, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00058164, "policy/clipfrac_avg": 0.00424064, "loss/policy_avg": -0.04389418, "loss/value_avg": 0.5461421, "val/clipfrac_avg": 0.24988537, "policy/entropy_avg": 0.15207291, "val/ratio": 0.99328047, "val/ratio_var": 4.207e-05, "val/num_eos_tokens": 497, "lr": 9.38e-06, "episode": 4032, "epoch": 0.20363636, "global_step/max_steps": "252/1238", "percentage": "20.36%", "elapsed_time": "2h 52m 43s", "remaining_time": "11h 15m 49s"} +{"eps": 0, "objective/kl": 11.8256588, "objective/entropy": -88.22281647, "objective/non_score_reward": -0.59128302, "objective/rlhf_reward": 8.1430912, "objective/scores": 8.75, "policy/approxkl_avg": 0.00085856, "policy/clipfrac_avg": 0.01591934, "loss/policy_avg": 0.31844419, "loss/value_avg": 0.80397522, "val/clipfrac_avg": 0.05827297, "policy/entropy_avg": 0.13445878, "val/ratio": 0.98985016, "val/ratio_var": 6.265e-05, "val/num_eos_tokens": 746, "lr": 9.37e-06, "episode": 4048, "epoch": 0.20444444, "global_step/max_steps": "253/1238", "percentage": "20.44%", "elapsed_time": "2h 53m 22s", "remaining_time": "11h 15m 1s"} +{"eps": 0, "objective/kl": 14.88226128, "objective/entropy": -42.02299881, "objective/non_score_reward": -0.74411303, "objective/rlhf_reward": 8.10744953, "objective/scores": 8.875, "policy/approxkl_avg": 0.00098294, "policy/clipfrac_avg": 0.01986194, "loss/policy_avg": 0.06932682, "loss/value_avg": 0.47045055, "val/clipfrac_avg": 0.09604226, "policy/entropy_avg": 0.18294621, "val/ratio": 0.98775363, "val/ratio_var": 5.442e-05, "val/num_eos_tokens": 202, "lr": 9.36e-06, "episode": 4064, "epoch": 0.20525253, "global_step/max_steps": "254/1238", "percentage": "20.52%", "elapsed_time": "2h 54m 6s", "remaining_time": "11h 14m 28s"} +{"eps": 0, "objective/kl": 15.1350069, "objective/entropy": -132.06040955, "objective/non_score_reward": -0.7567504, "objective/rlhf_reward": 8.19246864, "objective/scores": 8.9375, "policy/approxkl_avg": 0.00077902, "policy/clipfrac_avg": 0.01918134, "loss/policy_avg": 0.14544922, "loss/value_avg": 0.25122333, "val/clipfrac_avg": 0.08042471, "policy/entropy_avg": 0.1305809, "val/ratio": 0.99079823, "val/ratio_var": 3.779e-05, "val/num_eos_tokens": 424, "lr": 9.36e-06, "episode": 4080, "epoch": 0.20606061, "global_step/max_steps": "255/1238", "percentage": "20.60%", "elapsed_time": "2h 54m 43s", "remaining_time": "11h 13m 32s"} +{"eps": 0, "objective/kl": 17.12119484, "objective/entropy": -94.29174805, "objective/non_score_reward": -0.85605979, "objective/rlhf_reward": 8.67128372, "objective/scores": 9.5, "policy/approxkl_avg": 0.00090063, "policy/clipfrac_avg": 0.00575554, "loss/policy_avg": -0.18705142, "loss/value_avg": 0.85620797, "val/clipfrac_avg": 0.09692854, "policy/entropy_avg": 0.17468429, "val/ratio": 0.98858482, "val/ratio_var": 4.237e-05, "val/num_eos_tokens": 798, "lr": 9.35e-06, "episode": 4096, "epoch": 0.20686869, "global_step/max_steps": "256/1238", "percentage": "20.68%", "elapsed_time": "2h 55m 24s", "remaining_time": "11h 12m 49s"} +{"eps": 0, "objective/kl": 19.60912704, "objective/entropy": -157.23556519, "objective/non_score_reward": -0.98045635, "objective/rlhf_reward": 7.66016817, "objective/scores": 8.625, "policy/approxkl_avg": 0.00073781, "policy/clipfrac_avg": 0.0094704, "loss/policy_avg": -0.66018814, "loss/value_avg": 1.85427237, "val/clipfrac_avg": 0.17326012, "policy/entropy_avg": 0.17036653, "val/ratio": 0.99064994, "val/ratio_var": 5.125e-05, "val/num_eos_tokens": 383, "lr": 9.34e-06, "episode": 4112, "epoch": 0.20767677, "global_step/max_steps": "257/1238", "percentage": "20.76%", "elapsed_time": "2h 57m 7s", "remaining_time": "11h 16m 4s"} +{"eps": 0, "objective/kl": 11.77539062, "objective/entropy": -95.90520477, "objective/non_score_reward": -0.58876956, "objective/rlhf_reward": 7.54404306, "objective/scores": 8.125, "policy/approxkl_avg": 0.000771, "policy/clipfrac_avg": 0.00871011, "loss/policy_avg": 0.00069687, "loss/value_avg": 0.57274979, "val/clipfrac_avg": 0.20523447, "policy/entropy_avg": 0.13333511, "val/ratio": 0.99073422, "val/ratio_var": 5.306e-05, "val/num_eos_tokens": 553, "lr": 9.34e-06, "episode": 4128, "epoch": 0.20848485, "global_step/max_steps": "258/1238", "percentage": "20.84%", "elapsed_time": "2h 57m 37s", "remaining_time": "11h 14m 43s"} +{"eps": 0, "objective/kl": 15.8824482, "objective/entropy": -180.31079102, "objective/non_score_reward": -0.79412234, "objective/rlhf_reward": 7.74884653, "objective/scores": 8.5625, "policy/approxkl_avg": 0.00070872, "policy/clipfrac_avg": 0.00898685, "loss/policy_avg": -0.11990015, "loss/value_avg": 0.18013015, "val/clipfrac_avg": 0.14693329, "policy/entropy_avg": 0.14745712, "val/ratio": 0.99118769, "val/ratio_var": 5.386e-05, "val/num_eos_tokens": 400, "lr": 9.33e-06, "episode": 4144, "epoch": 0.20929293, "global_step/max_steps": "259/1238", "percentage": "20.92%", "elapsed_time": "2h 58m 35s", "remaining_time": "11h 15m 3s"} +{"eps": 0, "objective/kl": 11.50456524, "objective/entropy": -97.9331665, "objective/non_score_reward": -0.57522827, "objective/rlhf_reward": 10.41012383, "objective/scores": 11.0, "policy/approxkl_avg": 0.00104852, "policy/clipfrac_avg": 0.02218588, "loss/policy_avg": 0.04649061, "loss/value_avg": 1.06199384, "val/clipfrac_avg": 0.28776792, "policy/entropy_avg": 0.1638912, "val/ratio": 0.98948896, "val/ratio_var": 7.833e-05, "val/num_eos_tokens": 62, "lr": 9.32e-06, "episode": 4160, "epoch": 0.21010101, "global_step/max_steps": "260/1238", "percentage": "21.00%", "elapsed_time": "2h 59m 15s", "remaining_time": "11h 14m 18s"} +{"eps": 0, "objective/kl": 14.00249386, "objective/entropy": -75.23338318, "objective/non_score_reward": -0.70012474, "objective/rlhf_reward": 10.02643776, "objective/scores": 10.75, "policy/approxkl_avg": 0.00098499, "policy/clipfrac_avg": 0.00667524, "loss/policy_avg": -0.20035383, "loss/value_avg": 0.5576207, "val/clipfrac_avg": 0.16675466, "policy/entropy_avg": 0.17143822, "val/ratio": 0.98862988, "val/ratio_var": 6.113e-05, "val/num_eos_tokens": 11, "lr": 9.32e-06, "episode": 4176, "epoch": 0.21090909, "global_step/max_steps": "261/1238", "percentage": "21.08%", "elapsed_time": "2h 59m 48s", "remaining_time": "11h 13m 3s"} +{"eps": 0, "objective/kl": 14.50129604, "objective/entropy": -105.86523438, "objective/non_score_reward": -0.72506481, "objective/rlhf_reward": 10.21243477, "objective/scores": 10.9375, "policy/approxkl_avg": 0.00091697, "policy/clipfrac_avg": 0.01335984, "loss/policy_avg": 0.04166822, "loss/value_avg": 1.01220751, "val/clipfrac_avg": 0.11749287, "policy/entropy_avg": 0.1592083, "val/ratio": 0.98921061, "val/ratio_var": 5.974e-05, "val/num_eos_tokens": 54, "lr": 9.31e-06, "episode": 4192, "epoch": 0.21171717, "global_step/max_steps": "262/1238", "percentage": "21.16%", "elapsed_time": "3h 0m 21s", "remaining_time": "11h 11m 51s"} +{"eps": 0, "objective/kl": 12.70427895, "objective/entropy": -108.76235199, "objective/non_score_reward": -0.63521403, "objective/rlhf_reward": 8.8140049, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00076224, "policy/clipfrac_avg": 0.01205964, "loss/policy_avg": -0.12460523, "loss/value_avg": 0.35605776, "val/clipfrac_avg": 0.08863926, "policy/entropy_avg": 0.13946581, "val/ratio": 0.99143529, "val/ratio_var": 3.335e-05, "val/num_eos_tokens": 196, "lr": 9.3e-06, "episode": 4208, "epoch": 0.21252525, "global_step/max_steps": "263/1238", "percentage": "21.24%", "elapsed_time": "3h 1m 3s", "remaining_time": "11h 11m 15s"} +{"eps": 0, "objective/kl": 11.76550579, "objective/entropy": -82.82820129, "objective/non_score_reward": -0.58827525, "objective/rlhf_reward": 9.84141254, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00069419, "policy/clipfrac_avg": 0.0072274, "loss/policy_avg": 0.30006385, "loss/value_avg": 3.2313745, "val/clipfrac_avg": 0.07290898, "policy/entropy_avg": 0.12492609, "val/ratio": 0.99191201, "val/ratio_var": 3.886e-05, "val/num_eos_tokens": 336, "lr": 9.3e-06, "episode": 4224, "epoch": 0.21333333, "global_step/max_steps": "264/1238", "percentage": "21.32%", "elapsed_time": "3h 1m 38s", "remaining_time": "11h 10m 7s"} +{"eps": 0, "objective/kl": 15.94631004, "objective/entropy": -161.09799194, "objective/non_score_reward": -0.79731548, "objective/rlhf_reward": 8.1714344, "objective/scores": 9.0, "policy/approxkl_avg": 0.00078481, "policy/clipfrac_avg": 0.00563935, "loss/policy_avg": 0.1362697, "loss/value_avg": 0.25763941, "val/clipfrac_avg": 0.03482377, "policy/entropy_avg": 0.14366579, "val/ratio": 0.99078387, "val/ratio_var": 4.943e-05, "val/num_eos_tokens": 285, "lr": 9.29e-06, "episode": 4240, "epoch": 0.21414141, "global_step/max_steps": "265/1238", "percentage": "21.41%", "elapsed_time": "3h 2m 19s", "remaining_time": "11h 9m 27s"} +{"eps": 0, "objective/kl": 18.60851669, "objective/entropy": -38.92912674, "objective/non_score_reward": -0.93042582, "objective/rlhf_reward": 9.49926186, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00105092, "policy/clipfrac_avg": 0.01376679, "loss/policy_avg": -0.14799899, "loss/value_avg": 1.17362559, "val/clipfrac_avg": 0.16095665, "policy/entropy_avg": 0.17187881, "val/ratio": 0.99151725, "val/ratio_var": 0.00022518, "val/num_eos_tokens": 39, "lr": 9.28e-06, "episode": 4256, "epoch": 0.21494949, "global_step/max_steps": "266/1238", "percentage": "21.49%", "elapsed_time": "3h 3m 8s", "remaining_time": "11h 9m 13s"} +{"eps": 0, "objective/kl": 12.63461781, "objective/entropy": -55.39912415, "objective/non_score_reward": -0.63173091, "objective/rlhf_reward": 9.55576897, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00090836, "policy/clipfrac_avg": 0.00477819, "loss/policy_avg": -0.10845097, "loss/value_avg": 1.35971332, "val/clipfrac_avg": 0.17677703, "policy/entropy_avg": 0.11792755, "val/ratio": 0.99299872, "val/ratio_var": 0.00012306, "val/num_eos_tokens": 131, "lr": 9.28e-06, "episode": 4272, "epoch": 0.21575758, "global_step/max_steps": "267/1238", "percentage": "21.57%", "elapsed_time": "3h 3m 43s", "remaining_time": "11h 8m 10s"} +{"eps": 0, "objective/kl": 15.50401402, "objective/entropy": -59.65185165, "objective/non_score_reward": -0.77520066, "objective/rlhf_reward": 8.39276791, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00093208, "policy/clipfrac_avg": 0.00602042, "loss/policy_avg": -0.40072307, "loss/value_avg": 0.62740004, "val/clipfrac_avg": 0.24285002, "policy/entropy_avg": 0.16248131, "val/ratio": 0.98898256, "val/ratio_var": 4.561e-05, "val/num_eos_tokens": 26, "lr": 9.27e-06, "episode": 4288, "epoch": 0.21656566, "global_step/max_steps": "268/1238", "percentage": "21.65%", "elapsed_time": "3h 4m 26s", "remaining_time": "11h 7m 33s"} +{"eps": 0, "objective/kl": 17.36513519, "objective/entropy": -117.23480225, "objective/non_score_reward": -0.86825681, "objective/rlhf_reward": 8.83486843, "objective/scores": 9.6875, "policy/approxkl_avg": 0.00088191, "policy/clipfrac_avg": 0.0150713, "loss/policy_avg": -0.47263253, "loss/value_avg": 1.08739436, "val/clipfrac_avg": 0.06512845, "policy/entropy_avg": 0.16169071, "val/ratio": 0.98957515, "val/ratio_var": 6.196e-05, "val/num_eos_tokens": 367, "lr": 9.26e-06, "episode": 4304, "epoch": 0.21737374, "global_step/max_steps": "269/1238", "percentage": "21.73%", "elapsed_time": "3h 5m 1s", "remaining_time": "11h 6m 31s"} +{"eps": 0, "objective/kl": 14.76534653, "objective/entropy": -90.55671692, "objective/non_score_reward": -0.7382673, "objective/rlhf_reward": 8.5664196, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00078241, "policy/clipfrac_avg": 0.00568741, "loss/policy_avg": -0.29738408, "loss/value_avg": 0.61330676, "val/clipfrac_avg": 0.12544228, "policy/entropy_avg": 0.14926571, "val/ratio": 0.99037349, "val/ratio_var": 4.684e-05, "val/num_eos_tokens": 217, "lr": 9.25e-06, "episode": 4320, "epoch": 0.21818182, "global_step/max_steps": "270/1238", "percentage": "21.81%", "elapsed_time": "3h 5m 41s", "remaining_time": "11h 5m 45s"} +{"eps": 0, "objective/kl": 15.87863064, "objective/entropy": -108.28328705, "objective/non_score_reward": -0.79393148, "objective/rlhf_reward": 7.12794352, "objective/scores": 7.9375, "policy/approxkl_avg": 0.00077467, "policy/clipfrac_avg": 0.02020359, "loss/policy_avg": 0.10686541, "loss/value_avg": 0.35223833, "val/clipfrac_avg": 0.11016659, "policy/entropy_avg": 0.15306282, "val/ratio": 0.99145865, "val/ratio_var": 4.422e-05, "val/num_eos_tokens": 448, "lr": 9.25e-06, "episode": 4336, "epoch": 0.2189899, "global_step/max_steps": "271/1238", "percentage": "21.89%", "elapsed_time": "3h 6m 20s", "remaining_time": "11h 4m 53s"} +{"eps": 0, "objective/kl": 16.3353157, "objective/entropy": -133.5970459, "objective/non_score_reward": -0.81676579, "objective/rlhf_reward": 8.41760921, "objective/scores": 9.25, "policy/approxkl_avg": 0.00068241, "policy/clipfrac_avg": 0.00392046, "loss/policy_avg": -0.13800494, "loss/value_avg": 1.30031359, "val/clipfrac_avg": 0.15138726, "policy/entropy_avg": 0.12821245, "val/ratio": 0.99200106, "val/ratio_var": 4.987e-05, "val/num_eos_tokens": 1032, "lr": 9.24e-06, "episode": 4352, "epoch": 0.21979798, "global_step/max_steps": "272/1238", "percentage": "21.97%", "elapsed_time": "3h 7m 26s", "remaining_time": "11h 5m 42s"} +{"eps": 0, "objective/kl": 15.89100361, "objective/entropy": -69.04071808, "objective/non_score_reward": -0.79455018, "objective/rlhf_reward": 8.23279381, "objective/scores": 9.0, "policy/approxkl_avg": 0.00094351, "policy/clipfrac_avg": 0.00690113, "loss/policy_avg": -0.41694945, "loss/value_avg": 1.58386374, "val/clipfrac_avg": 0.15208563, "policy/entropy_avg": 0.17545414, "val/ratio": 0.98816669, "val/ratio_var": 9.141e-05, "val/num_eos_tokens": 95, "lr": 9.23e-06, "episode": 4368, "epoch": 0.22060606, "global_step/max_steps": "273/1238", "percentage": "22.05%", "elapsed_time": "3h 8m 1s", "remaining_time": "11h 4m 37s"} +{"eps": 0, "objective/kl": 14.84071159, "objective/entropy": -30.56391335, "objective/non_score_reward": -0.74203563, "objective/rlhf_reward": 9.75015259, "objective/scores": 10.5, "policy/approxkl_avg": 0.00089962, "policy/clipfrac_avg": 0.00418403, "loss/policy_avg": -0.23135731, "loss/value_avg": 1.03668344, "val/clipfrac_avg": 0.09715649, "policy/entropy_avg": 0.16372395, "val/ratio": 0.98903799, "val/ratio_var": 5.084e-05, "val/num_eos_tokens": 201, "lr": 9.23e-06, "episode": 4384, "epoch": 0.22141414, "global_step/max_steps": "274/1238", "percentage": "22.13%", "elapsed_time": "3h 8m 37s", "remaining_time": "11h 3m 38s"} +{"eps": 0, "objective/kl": 19.29312134, "objective/entropy": -208.77534485, "objective/non_score_reward": -0.96465611, "objective/rlhf_reward": 8.42987537, "objective/scores": 9.375, "policy/approxkl_avg": 0.00103967, "policy/clipfrac_avg": 0.02184339, "loss/policy_avg": -0.00809199, "loss/value_avg": 0.75641006, "val/clipfrac_avg": 0.11226044, "policy/entropy_avg": 0.13889122, "val/ratio": 0.99088442, "val/ratio_var": 4.535e-05, "val/num_eos_tokens": 316, "lr": 9.22e-06, "episode": 4400, "epoch": 0.22222222, "global_step/max_steps": "275/1238", "percentage": "22.21%", "elapsed_time": "3h 10m 18s", "remaining_time": "11h 6m 26s"} +{"eps": 0, "objective/kl": 12.33164787, "objective/entropy": -111.80735779, "objective/non_score_reward": -0.61658239, "objective/rlhf_reward": 11.74279308, "objective/scores": 12.375, "policy/approxkl_avg": 0.00080459, "policy/clipfrac_avg": 0.00896491, "loss/policy_avg": -0.01235494, "loss/value_avg": 1.01184821, "val/clipfrac_avg": 0.18904702, "policy/entropy_avg": 0.14202404, "val/ratio": 0.99086148, "val/ratio_var": 6.562e-05, "val/num_eos_tokens": 618, "lr": 9.21e-06, "episode": 4416, "epoch": 0.2230303, "global_step/max_steps": "276/1238", "percentage": "22.29%", "elapsed_time": "3h 11m 0s", "remaining_time": "11h 5m 45s"} +{"eps": 0, "objective/kl": 16.08571243, "objective/entropy": -201.3074646, "objective/non_score_reward": -0.80428565, "objective/rlhf_reward": 10.59024525, "objective/scores": 11.375, "policy/approxkl_avg": 0.0007445, "policy/clipfrac_avg": 0.01038559, "loss/policy_avg": -0.44561365, "loss/value_avg": 4.01507807, "val/clipfrac_avg": 0.10327339, "policy/entropy_avg": 0.13581823, "val/ratio": 0.9907583, "val/ratio_var": 6.719e-05, "val/num_eos_tokens": 695, "lr": 9.2e-06, "episode": 4432, "epoch": 0.22383838, "global_step/max_steps": "277/1238", "percentage": "22.37%", "elapsed_time": "3h 11m 57s", "remaining_time": "11h 5m 58s"} +{"eps": 0, "objective/kl": 19.13482857, "objective/entropy": -81.09490967, "objective/non_score_reward": -0.95674145, "objective/rlhf_reward": 8.37138367, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00092073, "policy/clipfrac_avg": 0.00746871, "loss/policy_avg": -0.08938003, "loss/value_avg": 0.3082343, "val/clipfrac_avg": 0.1193381, "policy/entropy_avg": 0.16699982, "val/ratio": 0.98897159, "val/ratio_var": 3.133e-05, "val/num_eos_tokens": 70, "lr": 9.2e-06, "episode": 4448, "epoch": 0.22464646, "global_step/max_steps": "278/1238", "percentage": "22.46%", "elapsed_time": "3h 12m 33s", "remaining_time": "11h 4m 57s"} +{"eps": 0, "objective/kl": 12.76495934, "objective/entropy": -138.91981506, "objective/non_score_reward": -0.63824809, "objective/rlhf_reward": 9.53362656, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0007106, "policy/clipfrac_avg": 0.01297901, "loss/policy_avg": -0.47569281, "loss/value_avg": 0.47323668, "val/clipfrac_avg": 0.08647473, "policy/entropy_avg": 0.11971688, "val/ratio": 0.99265003, "val/ratio_var": 3.489e-05, "val/num_eos_tokens": 596, "lr": 9.19e-06, "episode": 4464, "epoch": 0.22545455, "global_step/max_steps": "279/1238", "percentage": "22.54%", "elapsed_time": "3h 13m 11s", "remaining_time": "11h 4m 2s"} +{"eps": 0, "objective/kl": 16.78522873, "objective/entropy": -100.83489227, "objective/non_score_reward": -0.83926141, "objective/rlhf_reward": 8.42636395, "objective/scores": 9.25, "policy/approxkl_avg": 0.00086147, "policy/clipfrac_avg": 0.00798733, "loss/policy_avg": 0.04888364, "loss/value_avg": 0.30278966, "val/clipfrac_avg": 0.04620703, "policy/entropy_avg": 0.13911915, "val/ratio": 0.99057651, "val/ratio_var": 2.736e-05, "val/num_eos_tokens": 27, "lr": 9.18e-06, "episode": 4480, "epoch": 0.22626263, "global_step/max_steps": "280/1238", "percentage": "22.62%", "elapsed_time": "3h 13m 49s", "remaining_time": "11h 3m 9s"} +{"eps": 0, "objective/kl": 17.81083679, "objective/entropy": -116.21156311, "objective/non_score_reward": -0.89054179, "objective/rlhf_reward": 10.67195797, "objective/scores": 11.5625, "policy/approxkl_avg": 0.00076192, "policy/clipfrac_avg": 0.00715569, "loss/policy_avg": -0.31523934, "loss/value_avg": 0.84530139, "val/clipfrac_avg": 0.17372316, "policy/entropy_avg": 0.1354847, "val/ratio": 0.99127138, "val/ratio_var": 4.617e-05, "val/num_eos_tokens": 248, "lr": 9.18e-06, "episode": 4496, "epoch": 0.22707071, "global_step/max_steps": "281/1238", "percentage": "22.70%", "elapsed_time": "3h 14m 27s", "remaining_time": "11h 2m 17s"} +{"eps": 0, "objective/kl": 15.42947865, "objective/entropy": -123.32449341, "objective/non_score_reward": -0.771474, "objective/rlhf_reward": 9.44336891, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00077692, "policy/clipfrac_avg": 0.01072021, "loss/policy_avg": 0.29188928, "loss/value_avg": 1.00246286, "val/clipfrac_avg": 0.11208211, "policy/entropy_avg": 0.12940633, "val/ratio": 0.99126565, "val/ratio_var": 3.814e-05, "val/num_eos_tokens": 43, "lr": 9.17e-06, "episode": 4512, "epoch": 0.22787879, "global_step/max_steps": "282/1238", "percentage": "22.78%", "elapsed_time": "3h 15m 6s", "remaining_time": "11h 1m 24s"} +{"eps": 0, "objective/kl": 17.24679756, "objective/entropy": -79.49951172, "objective/non_score_reward": -0.86233985, "objective/rlhf_reward": 7.86812878, "objective/scores": 8.75, "policy/approxkl_avg": 0.00085698, "policy/clipfrac_avg": 0.02880413, "loss/policy_avg": -0.23349275, "loss/value_avg": 0.64826846, "val/clipfrac_avg": 0.14292857, "policy/entropy_avg": 0.13358784, "val/ratio": 0.9911713, "val/ratio_var": 4.618e-05, "val/num_eos_tokens": 444, "lr": 9.16e-06, "episode": 4528, "epoch": 0.22868687, "global_step/max_steps": "283/1238", "percentage": "22.86%", "elapsed_time": "3h 15m 47s", "remaining_time": "11h 0m 42s"} +{"eps": 0, "objective/kl": 22.31014442, "objective/entropy": -95.04164124, "objective/non_score_reward": -1.11550725, "objective/rlhf_reward": 8.86886787, "objective/scores": 10.0, "policy/approxkl_avg": 0.00104801, "policy/clipfrac_avg": 0.01095062, "loss/policy_avg": 0.12218322, "loss/value_avg": 0.19281775, "val/clipfrac_avg": 0.13048363, "policy/entropy_avg": 0.16732121, "val/ratio": 0.98909581, "val/ratio_var": 4.824e-05, "val/num_eos_tokens": 203, "lr": 9.15e-06, "episode": 4544, "epoch": 0.22949495, "global_step/max_steps": "284/1238", "percentage": "22.94%", "elapsed_time": "3h 16m 37s", "remaining_time": "11h 0m 27s"} +{"eps": 0, "objective/kl": 20.04546738, "objective/entropy": -42.42488098, "objective/non_score_reward": -1.00227356, "objective/rlhf_reward": 7.65397644, "objective/scores": 8.625, "policy/approxkl_avg": 0.00088487, "policy/clipfrac_avg": 0.00740264, "loss/policy_avg": -0.00236215, "loss/value_avg": 0.58660454, "val/clipfrac_avg": 0.16762601, "policy/entropy_avg": 0.1446681, "val/ratio": 0.98966211, "val/ratio_var": 2.382e-05, "val/num_eos_tokens": 66, "lr": 9.15e-06, "episode": 4560, "epoch": 0.23030303, "global_step/max_steps": "285/1238", "percentage": "23.02%", "elapsed_time": "3h 17m 16s", "remaining_time": "10h 59m 40s"} +{"eps": 0, "objective/kl": 16.77660561, "objective/entropy": -106.9161377, "objective/non_score_reward": -0.83883035, "objective/rlhf_reward": 8.2041378, "objective/scores": 9.0625, "policy/approxkl_avg": 0.00081328, "policy/clipfrac_avg": 0.00947195, "loss/policy_avg": -0.56265348, "loss/value_avg": 1.47391558, "val/clipfrac_avg": 0.19085138, "policy/entropy_avg": 0.14436913, "val/ratio": 0.98970473, "val/ratio_var": 4.601e-05, "val/num_eos_tokens": 222, "lr": 9.14e-06, "episode": 4576, "epoch": 0.23111111, "global_step/max_steps": "286/1238", "percentage": "23.10%", "elapsed_time": "3h 17m 48s", "remaining_time": "10h 58m 27s"} +{"eps": 0, "objective/kl": 13.76268864, "objective/entropy": -144.25914001, "objective/non_score_reward": -0.68813443, "objective/rlhf_reward": 9.99155331, "objective/scores": 10.6875, "policy/approxkl_avg": 0.00205441, "policy/clipfrac_avg": 0.0090272, "loss/policy_avg": 0.10144818, "loss/value_avg": 3.8196106, "val/clipfrac_avg": 0.21551818, "policy/entropy_avg": 0.13915539, "val/ratio": 0.99080622, "val/ratio_var": 0.00011211, "val/num_eos_tokens": 491, "lr": 9.13e-06, "episode": 4592, "epoch": 0.23191919, "global_step/max_steps": "287/1238", "percentage": "23.18%", "elapsed_time": "3h 18m 32s", "remaining_time": "10h 57m 52s"} +{"eps": 0, "objective/kl": 18.2749176, "objective/entropy": -86.54766083, "objective/non_score_reward": -0.91374588, "objective/rlhf_reward": 8.64094162, "objective/scores": 9.5625, "policy/approxkl_avg": 0.00102439, "policy/clipfrac_avg": 0.01274296, "loss/policy_avg": -0.14434215, "loss/value_avg": 1.03019476, "val/clipfrac_avg": 0.0906482, "policy/entropy_avg": 0.17115402, "val/ratio": 0.98882735, "val/ratio_var": 6.464e-05, "val/num_eos_tokens": 80, "lr": 9.12e-06, "episode": 4608, "epoch": 0.23272727, "global_step/max_steps": "288/1238", "percentage": "23.26%", "elapsed_time": "3h 19m 7s", "remaining_time": "10h 56m 50s"} +{"eps": 0, "objective/kl": 19.72304153, "objective/entropy": -80.63391113, "objective/non_score_reward": -0.98615211, "objective/rlhf_reward": 9.89275455, "objective/scores": 10.875, "policy/approxkl_avg": 0.0009435, "policy/clipfrac_avg": 0.00978278, "loss/policy_avg": -0.22777478, "loss/value_avg": 0.49742779, "val/clipfrac_avg": 0.11158282, "policy/entropy_avg": 0.14695454, "val/ratio": 0.9896304, "val/ratio_var": 4.675e-05, "val/num_eos_tokens": 422, "lr": 9.12e-06, "episode": 4624, "epoch": 0.23353535, "global_step/max_steps": "289/1238", "percentage": "23.34%", "elapsed_time": "3h 19m 52s", "remaining_time": "10h 56m 21s"} +{"eps": 0, "objective/kl": 13.74578857, "objective/entropy": -90.72955322, "objective/non_score_reward": -0.68728948, "objective/rlhf_reward": 9.58614731, "objective/scores": 10.25, "policy/approxkl_avg": 0.00078291, "policy/clipfrac_avg": 0.00626225, "loss/policy_avg": -0.42491421, "loss/value_avg": 1.53578544, "val/clipfrac_avg": 0.20132911, "policy/entropy_avg": 0.13948059, "val/ratio": 0.99057132, "val/ratio_var": 4.521e-05, "val/num_eos_tokens": 98, "lr": 9.11e-06, "episode": 4640, "epoch": 0.23434343, "global_step/max_steps": "290/1238", "percentage": "23.42%", "elapsed_time": "3h 20m 33s", "remaining_time": "10h 55m 36s"} +{"eps": 0, "objective/kl": 18.2149601, "objective/entropy": -127.00852203, "objective/non_score_reward": -0.910748, "objective/rlhf_reward": 9.72987652, "objective/scores": 10.625, "policy/approxkl_avg": 0.00093915, "policy/clipfrac_avg": 0.01690863, "loss/policy_avg": -0.0096181, "loss/value_avg": 1.82759142, "val/clipfrac_avg": 0.18527053, "policy/entropy_avg": 0.14864182, "val/ratio": 0.990785, "val/ratio_var": 4.85e-05, "val/num_eos_tokens": 312, "lr": 9.1e-06, "episode": 4656, "epoch": 0.23515152, "global_step/max_steps": "291/1238", "percentage": "23.51%", "elapsed_time": "3h 21m 15s", "remaining_time": "10h 54m 56s"} +{"eps": 0, "objective/kl": 17.06829643, "objective/entropy": -121.64453888, "objective/non_score_reward": -0.85341483, "objective/rlhf_reward": 9.34971046, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00109892, "policy/clipfrac_avg": 0.01564177, "loss/policy_avg": -0.31738281, "loss/value_avg": 0.70548254, "val/clipfrac_avg": 0.12627627, "policy/entropy_avg": 0.13405228, "val/ratio": 0.99253559, "val/ratio_var": 4.043e-05, "val/num_eos_tokens": 655, "lr": 9.09e-06, "episode": 4672, "epoch": 0.2359596, "global_step/max_steps": "292/1238", "percentage": "23.59%", "elapsed_time": "3h 21m 55s", "remaining_time": "10h 54m 11s"} +{"eps": 0, "objective/kl": 15.00445557, "objective/entropy": -96.61104584, "objective/non_score_reward": -0.7502228, "objective/rlhf_reward": 8.16384029, "objective/scores": 8.9375, "policy/approxkl_avg": 0.00080205, "policy/clipfrac_avg": 0.00355702, "loss/policy_avg": 0.25364614, "loss/value_avg": 0.84482312, "val/clipfrac_avg": 0.22700234, "policy/entropy_avg": 0.14197516, "val/ratio": 0.98927867, "val/ratio_var": 0.00012587, "val/num_eos_tokens": 365, "lr": 9.09e-06, "episode": 4688, "epoch": 0.23676768, "global_step/max_steps": "293/1238", "percentage": "23.67%", "elapsed_time": "3h 22m 32s", "remaining_time": "10h 53m 14s"} +{"eps": 0, "objective/kl": 16.63157654, "objective/entropy": -112.97701263, "objective/non_score_reward": -0.83157885, "objective/rlhf_reward": 12.01217079, "objective/scores": 12.875, "policy/approxkl_avg": 0.00069373, "policy/clipfrac_avg": 0.00447247, "loss/policy_avg": -0.48008922, "loss/value_avg": 0.60219741, "val/clipfrac_avg": 0.3252812, "policy/entropy_avg": 0.10819668, "val/ratio": 0.99273801, "val/ratio_var": 4.103e-05, "val/num_eos_tokens": 138, "lr": 9.08e-06, "episode": 4704, "epoch": 0.23757576, "global_step/max_steps": "294/1238", "percentage": "23.75%", "elapsed_time": "3h 23m 7s", "remaining_time": "10h 52m 11s"} +{"eps": 0, "objective/kl": 15.85359955, "objective/entropy": -9.15427589, "objective/non_score_reward": -0.79267997, "objective/rlhf_reward": 9.29325771, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00080397, "policy/clipfrac_avg": 0.00632806, "loss/policy_avg": 0.05609272, "loss/value_avg": 0.51578999, "val/clipfrac_avg": 0.19731221, "policy/entropy_avg": 0.12773895, "val/ratio": 0.99084854, "val/ratio_var": 4.185e-05, "val/num_eos_tokens": 17, "lr": 9.07e-06, "episode": 4720, "epoch": 0.23838384, "global_step/max_steps": "295/1238", "percentage": "23.83%", "elapsed_time": "3h 23m 42s", "remaining_time": "10h 51m 12s"} +{"eps": 0, "objective/kl": 13.81769371, "objective/entropy": -76.3034668, "objective/non_score_reward": -0.69088471, "objective/rlhf_reward": 10.17630291, "objective/scores": 10.875, "policy/approxkl_avg": 0.000838, "policy/clipfrac_avg": 0.01853689, "loss/policy_avg": -0.56606609, "loss/value_avg": 1.65913534, "val/clipfrac_avg": 0.3154422, "policy/entropy_avg": 0.12799948, "val/ratio": 0.99212801, "val/ratio_var": 4.132e-05, "val/num_eos_tokens": 343, "lr": 9.06e-06, "episode": 4736, "epoch": 0.23919192, "global_step/max_steps": "296/1238", "percentage": "23.91%", "elapsed_time": "3h 24m 20s", "remaining_time": "10h 50m 16s"} +{"eps": 0, "objective/kl": 22.04182053, "objective/entropy": -127.31668854, "objective/non_score_reward": -1.10209084, "objective/rlhf_reward": 7.94868994, "objective/scores": 9.0625, "policy/approxkl_avg": 0.00087046, "policy/clipfrac_avg": 0.00863572, "loss/policy_avg": -0.13689612, "loss/value_avg": 0.22567093, "val/clipfrac_avg": 0.06009959, "policy/entropy_avg": 0.14861584, "val/ratio": 0.98964065, "val/ratio_var": 3.544e-05, "val/num_eos_tokens": 572, "lr": 9.05e-06, "episode": 4752, "epoch": 0.24, "global_step/max_steps": "297/1238", "percentage": "23.99%", "elapsed_time": "3h 25m 15s", "remaining_time": "10h 50m 19s"} +{"eps": 0, "objective/kl": 17.81268692, "objective/entropy": -99.76239014, "objective/non_score_reward": -0.89063436, "objective/rlhf_reward": 10.42186546, "objective/scores": 11.3125, "policy/approxkl_avg": 0.00076316, "policy/clipfrac_avg": 0.01034308, "loss/policy_avg": -0.28741693, "loss/value_avg": 1.40587449, "val/clipfrac_avg": 0.16891876, "policy/entropy_avg": 0.12606573, "val/ratio": 0.99156499, "val/ratio_var": 3.899e-05, "val/num_eos_tokens": 157, "lr": 9.05e-06, "episode": 4768, "epoch": 0.24080808, "global_step/max_steps": "298/1238", "percentage": "24.07%", "elapsed_time": "3h 25m 50s", "remaining_time": "10h 49m 17s"} +{"eps": 0, "objective/kl": 18.92072487, "objective/entropy": -53.7321167, "objective/non_score_reward": -0.94603622, "objective/rlhf_reward": 8.97974586, "objective/scores": 9.9375, "policy/approxkl_avg": 0.00087351, "policy/clipfrac_avg": 0.00515089, "loss/policy_avg": -0.33470079, "loss/value_avg": 1.45772445, "val/clipfrac_avg": 0.07536731, "policy/entropy_avg": 0.15177441, "val/ratio": 0.98990226, "val/ratio_var": 3.84e-05, "val/num_eos_tokens": 184, "lr": 9.04e-06, "episode": 4784, "epoch": 0.24161616, "global_step/max_steps": "299/1238", "percentage": "24.15%", "elapsed_time": "3h 26m 22s", "remaining_time": "10h 48m 7s"} +{"eps": 0, "objective/kl": 18.13817406, "objective/entropy": -111.41056824, "objective/non_score_reward": -0.90690863, "objective/rlhf_reward": 6.54230976, "objective/scores": 7.4375, "policy/approxkl_avg": 0.00087313, "policy/clipfrac_avg": 0.02135075, "loss/policy_avg": -0.16605234, "loss/value_avg": 0.31406185, "val/clipfrac_avg": 0.05479028, "policy/entropy_avg": 0.12722576, "val/ratio": 0.99155056, "val/ratio_var": 4.507e-05, "val/num_eos_tokens": 420, "lr": 9.03e-06, "episode": 4800, "epoch": 0.24242424, "global_step/max_steps": "300/1238", "percentage": "24.23%", "elapsed_time": "3h 27m 12s", "remaining_time": "10h 47m 53s"} +{"eps": 0, "objective/kl": 20.15332985, "objective/entropy": -129.36647034, "objective/non_score_reward": -1.00766659, "objective/rlhf_reward": 9.46889591, "objective/scores": 10.5, "policy/approxkl_avg": 0.00088978, "policy/clipfrac_avg": 0.01343006, "loss/policy_avg": -0.03828721, "loss/value_avg": 0.31914866, "val/clipfrac_avg": 0.08865604, "policy/entropy_avg": 0.13002968, "val/ratio": 0.9906534, "val/ratio_var": 4.156e-05, "val/num_eos_tokens": 169, "lr": 9.02e-06, "episode": 4816, "epoch": 0.24323232, "global_step/max_steps": "301/1238", "percentage": "24.31%", "elapsed_time": "3h 28m 11s", "remaining_time": "10h 48m 4s"} +{"eps": 0, "objective/kl": 23.02680397, "objective/entropy": -88.79689026, "objective/non_score_reward": -1.15134025, "objective/rlhf_reward": 8.06740952, "objective/scores": 9.25, "policy/approxkl_avg": 0.0007821, "policy/clipfrac_avg": 0.00636195, "loss/policy_avg": -0.01393644, "loss/value_avg": 0.27543592, "val/clipfrac_avg": 0.08944192, "policy/entropy_avg": 0.1479187, "val/ratio": 0.99107575, "val/ratio_var": 2.418e-05, "val/num_eos_tokens": 586, "lr": 9.02e-06, "episode": 4832, "epoch": 0.2440404, "global_step/max_steps": "302/1238", "percentage": "24.39%", "elapsed_time": "3h 28m 52s", "remaining_time": "10h 47m 21s"} +{"eps": 0, "objective/kl": 24.88728523, "objective/entropy": -153.92424011, "objective/non_score_reward": -1.24436426, "objective/rlhf_reward": 7.62087011, "objective/scores": 8.875, "policy/approxkl_avg": 0.00107976, "policy/clipfrac_avg": 0.02070121, "loss/policy_avg": 0.16253716, "loss/value_avg": 0.39628071, "val/clipfrac_avg": 0.09651351, "policy/entropy_avg": 0.14539474, "val/ratio": 0.9895668, "val/ratio_var": 5.088e-05, "val/num_eos_tokens": 141, "lr": 9.01e-06, "episode": 4848, "epoch": 0.24484848, "global_step/max_steps": "303/1238", "percentage": "24.47%", "elapsed_time": "3h 29m 52s", "remaining_time": "10h 47m 37s"} +{"eps": 0, "objective/kl": 18.14400673, "objective/entropy": -112.62982178, "objective/non_score_reward": -0.90720034, "objective/rlhf_reward": 9.68264294, "objective/scores": 10.5625, "policy/approxkl_avg": 0.00081598, "policy/clipfrac_avg": 0.01846667, "loss/policy_avg": 0.33741572, "loss/value_avg": 3.72596812, "val/clipfrac_avg": 0.1720764, "policy/entropy_avg": 0.1183176, "val/ratio": 0.99209106, "val/ratio_var": 2.862e-05, "val/num_eos_tokens": 604, "lr": 9e-06, "episode": 4864, "epoch": 0.24565657, "global_step/max_steps": "304/1238", "percentage": "24.56%", "elapsed_time": "3h 30m 36s", "remaining_time": "10h 47m 3s"} +{"eps": 0, "objective/kl": 21.02062225, "objective/entropy": -110.0030365, "objective/non_score_reward": -1.05103111, "objective/rlhf_reward": 9.30053139, "objective/scores": 10.375, "policy/approxkl_avg": 0.00095496, "policy/clipfrac_avg": 0.0229989, "loss/policy_avg": 0.14597805, "loss/value_avg": 0.46867594, "val/clipfrac_avg": 0.1400592, "policy/entropy_avg": 0.1488108, "val/ratio": 0.98954076, "val/ratio_var": 4.94e-05, "val/num_eos_tokens": 730, "lr": 8.99e-06, "episode": 4880, "epoch": 0.24646465, "global_step/max_steps": "305/1238", "percentage": "24.64%", "elapsed_time": "3h 31m 18s", "remaining_time": "10h 46m 23s"} +{"eps": 0, "objective/kl": 19.59725952, "objective/entropy": -137.54716492, "objective/non_score_reward": -0.97986293, "objective/rlhf_reward": 8.93810558, "objective/scores": 9.9375, "policy/approxkl_avg": 0.00070753, "policy/clipfrac_avg": 0.0133344, "loss/policy_avg": -0.09728779, "loss/value_avg": 0.6937114, "val/clipfrac_avg": 0.25466752, "policy/entropy_avg": 0.12405539, "val/ratio": 0.99157512, "val/ratio_var": 4.609e-05, "val/num_eos_tokens": 467, "lr": 8.98e-06, "episode": 4896, "epoch": 0.24727273, "global_step/max_steps": "306/1238", "percentage": "24.72%", "elapsed_time": "3h 32m 9s", "remaining_time": "10h 46m 10s"} +{"eps": 0, "objective/kl": 16.86310196, "objective/entropy": -90.8353653, "objective/non_score_reward": -0.84315509, "objective/rlhf_reward": 10.84434509, "objective/scores": 11.6875, "policy/approxkl_avg": 0.00073147, "policy/clipfrac_avg": 0.00725281, "loss/policy_avg": 0.18694513, "loss/value_avg": 0.5242486, "val/clipfrac_avg": 0.25227472, "policy/entropy_avg": 0.15680981, "val/ratio": 0.99118817, "val/ratio_var": 4.908e-05, "val/num_eos_tokens": 632, "lr": 8.97e-06, "episode": 4912, "epoch": 0.24808081, "global_step/max_steps": "307/1238", "percentage": "24.80%", "elapsed_time": "3h 32m 47s", "remaining_time": "10h 45m 18s"} +{"eps": 0, "objective/kl": 19.8286705, "objective/entropy": -96.56021118, "objective/non_score_reward": -0.99143362, "objective/rlhf_reward": 8.7507534, "objective/scores": 9.75, "policy/approxkl_avg": 0.00120711, "policy/clipfrac_avg": 0.02772552, "loss/policy_avg": -0.05933833, "loss/value_avg": 0.71497875, "val/clipfrac_avg": 0.07701821, "policy/entropy_avg": 0.14550871, "val/ratio": 0.99026, "val/ratio_var": 5.601e-05, "val/num_eos_tokens": 384, "lr": 8.97e-06, "episode": 4928, "epoch": 0.24888889, "global_step/max_steps": "308/1238", "percentage": "24.88%", "elapsed_time": "3h 33m 21s", "remaining_time": "10h 44m 14s"} +{"eps": 0, "objective/kl": 21.63240623, "objective/entropy": -98.84840393, "objective/non_score_reward": -1.08162045, "objective/rlhf_reward": 11.06291103, "objective/scores": 12.125, "policy/approxkl_avg": 0.0009883, "policy/clipfrac_avg": 0.0213239, "loss/policy_avg": -0.45973384, "loss/value_avg": 0.95647049, "val/clipfrac_avg": 0.17546955, "policy/entropy_avg": 0.15676051, "val/ratio": 0.98993069, "val/ratio_var": 8.197e-05, "val/num_eos_tokens": 12, "lr": 8.96e-06, "episode": 4944, "epoch": 0.24969697, "global_step/max_steps": "309/1238", "percentage": "24.96%", "elapsed_time": "3h 34m 0s", "remaining_time": "10h 43m 24s"} +{"eps": 0, "objective/kl": 19.03496933, "objective/entropy": -54.2794838, "objective/non_score_reward": -0.95174849, "objective/rlhf_reward": 8.87637711, "objective/scores": 9.8125, "policy/approxkl_avg": 0.00124111, "policy/clipfrac_avg": 0.00543892, "loss/policy_avg": -0.03560265, "loss/value_avg": 1.04886663, "val/clipfrac_avg": 0.19135892, "policy/entropy_avg": 0.15647936, "val/ratio": 0.98850501, "val/ratio_var": 7.225e-05, "val/num_eos_tokens": 3, "lr": 8.95e-06, "episode": 4960, "epoch": 0.25050505, "global_step/max_steps": "310/1238", "percentage": "25.04%", "elapsed_time": "3h 34m 29s", "remaining_time": "10h 42m 6s"} +{"eps": 0, "objective/kl": 16.92250443, "objective/entropy": -90.46046448, "objective/non_score_reward": -0.84612536, "objective/rlhf_reward": 8.32575035, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00101591, "policy/clipfrac_avg": 0.01187141, "loss/policy_avg": 0.02286727, "loss/value_avg": 0.94079149, "val/clipfrac_avg": 0.22986652, "policy/entropy_avg": 0.15433311, "val/ratio": 0.99067533, "val/ratio_var": 6.914e-05, "val/num_eos_tokens": 578, "lr": 8.94e-06, "episode": 4976, "epoch": 0.25131313, "global_step/max_steps": "311/1238", "percentage": "25.12%", "elapsed_time": "3h 35m 3s", "remaining_time": "10h 41m 2s"} +{"eps": 0, "objective/kl": 18.02437401, "objective/entropy": -124.92307281, "objective/non_score_reward": -0.90121877, "objective/rlhf_reward": 9.07143784, "objective/scores": 10.0, "policy/approxkl_avg": 0.00075484, "policy/clipfrac_avg": 0.01587069, "loss/policy_avg": -0.29373524, "loss/value_avg": 0.73299807, "val/clipfrac_avg": 0.10427083, "policy/entropy_avg": 0.12101477, "val/ratio": 0.9913851, "val/ratio_var": 5.301e-05, "val/num_eos_tokens": 173, "lr": 8.93e-06, "episode": 4992, "epoch": 0.25212121, "global_step/max_steps": "312/1238", "percentage": "25.20%", "elapsed_time": "3h 35m 41s", "remaining_time": "10h 40m 10s"} +{"eps": 0, "objective/kl": 17.74087143, "objective/entropy": -52.38456345, "objective/non_score_reward": -0.8870436, "objective/rlhf_reward": 9.17155075, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00070875, "policy/clipfrac_avg": 0.00857322, "loss/policy_avg": 0.26103121, "loss/value_avg": 0.77205074, "val/clipfrac_avg": 0.16540912, "policy/entropy_avg": 0.12889361, "val/ratio": 0.99089134, "val/ratio_var": 1.972e-05, "val/num_eos_tokens": 54, "lr": 8.93e-06, "episode": 5008, "epoch": 0.25292929, "global_step/max_steps": "313/1238", "percentage": "25.28%", "elapsed_time": "3h 36m 18s", "remaining_time": "10h 39m 16s"} +{"eps": 0, "objective/kl": 20.90007591, "objective/entropy": -135.29092407, "objective/non_score_reward": -1.04500377, "objective/rlhf_reward": 9.15030861, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00069395, "policy/clipfrac_avg": 0.00606626, "loss/policy_avg": -0.00157588, "loss/value_avg": 0.2577588, "val/clipfrac_avg": 0.11059861, "policy/entropy_avg": 0.12897873, "val/ratio": 0.99136955, "val/ratio_var": 4.32e-05, "val/num_eos_tokens": 75, "lr": 8.92e-06, "episode": 5024, "epoch": 0.25373737, "global_step/max_steps": "314/1238", "percentage": "25.36%", "elapsed_time": "3h 37m 0s", "remaining_time": "10h 38m 33s"} +{"eps": 0, "objective/kl": 17.85972214, "objective/entropy": -87.8085556, "objective/non_score_reward": -0.89298612, "objective/rlhf_reward": 10.04841995, "objective/scores": 10.9375, "policy/approxkl_avg": 0.00102315, "policy/clipfrac_avg": 0.0056943, "loss/policy_avg": 0.13728793, "loss/value_avg": 1.64574456, "val/clipfrac_avg": 0.20232242, "policy/entropy_avg": 0.18284661, "val/ratio": 0.98796684, "val/ratio_var": 5.565e-05, "val/num_eos_tokens": 37, "lr": 8.91e-06, "episode": 5040, "epoch": 0.25454545, "global_step/max_steps": "315/1238", "percentage": "25.44%", "elapsed_time": "3h 37m 46s", "remaining_time": "10h 38m 8s"} +{"eps": 0, "objective/kl": 18.62837219, "objective/entropy": -81.0003891, "objective/non_score_reward": -0.93141866, "objective/rlhf_reward": 8.27951908, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00082067, "policy/clipfrac_avg": 0.00500072, "loss/policy_avg": -0.03426008, "loss/value_avg": 0.33592397, "val/clipfrac_avg": 0.08686407, "policy/entropy_avg": 0.14201808, "val/ratio": 0.99060595, "val/ratio_var": 4.339e-05, "val/num_eos_tokens": 117, "lr": 8.9e-06, "episode": 5056, "epoch": 0.25535354, "global_step/max_steps": "316/1238", "percentage": "25.53%", "elapsed_time": "3h 38m 25s", "remaining_time": "10h 37m 17s"} +{"eps": 0, "objective/kl": 18.40201569, "objective/entropy": -72.72438049, "objective/non_score_reward": -0.92010081, "objective/rlhf_reward": 9.06427383, "objective/scores": 10.0, "policy/approxkl_avg": 0.00280085, "policy/clipfrac_avg": 0.01839325, "loss/policy_avg": -0.08662297, "loss/value_avg": 0.64328486, "val/clipfrac_avg": 0.24806933, "policy/entropy_avg": 0.14298725, "val/ratio": 0.99010003, "val/ratio_var": 6.167e-05, "val/num_eos_tokens": 137, "lr": 8.89e-06, "episode": 5072, "epoch": 0.25616162, "global_step/max_steps": "317/1238", "percentage": "25.61%", "elapsed_time": "3h 39m 7s", "remaining_time": "10h 36m 37s"} +{"eps": 0, "objective/kl": 18.7077446, "objective/entropy": -108.95730591, "objective/non_score_reward": -0.93538725, "objective/rlhf_reward": 8.50211334, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00083053, "policy/clipfrac_avg": 0.02855648, "loss/policy_avg": -0.3129518, "loss/value_avg": 0.87961721, "val/clipfrac_avg": 0.08825132, "policy/entropy_avg": 0.13506407, "val/ratio": 0.99024236, "val/ratio_var": 4.728e-05, "val/num_eos_tokens": 314, "lr": 8.88e-06, "episode": 5088, "epoch": 0.2569697, "global_step/max_steps": "318/1238", "percentage": "25.69%", "elapsed_time": "3h 39m 46s", "remaining_time": "10h 35m 48s"} +{"eps": 0, "objective/kl": 22.0740509, "objective/entropy": -173.64369202, "objective/non_score_reward": -1.10370255, "objective/rlhf_reward": 7.71660995, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00095863, "policy/clipfrac_avg": 0.052637, "loss/policy_avg": 0.38284558, "loss/value_avg": 1.08547282, "val/clipfrac_avg": 0.18396008, "policy/entropy_avg": 0.13795662, "val/ratio": 0.98962194, "val/ratio_var": 5.952e-05, "val/num_eos_tokens": 46, "lr": 8.88e-06, "episode": 5104, "epoch": 0.25777778, "global_step/max_steps": "319/1238", "percentage": "25.77%", "elapsed_time": "3h 41m 33s", "remaining_time": "10h 38m 17s"} +{"eps": 0, "objective/kl": 15.28188992, "objective/entropy": -72.43736267, "objective/non_score_reward": -0.76409447, "objective/rlhf_reward": 11.64996815, "objective/scores": 12.4375, "policy/approxkl_avg": 0.00130854, "policy/clipfrac_avg": 0.03898612, "loss/policy_avg": -0.01253262, "loss/value_avg": 2.47177815, "val/clipfrac_avg": 0.35684174, "policy/entropy_avg": 0.16540432, "val/ratio": 0.98909771, "val/ratio_var": 0.00016938, "val/num_eos_tokens": 403, "lr": 8.87e-06, "episode": 5120, "epoch": 0.25858586, "global_step/max_steps": "320/1238", "percentage": "25.85%", "elapsed_time": "3h 42m 2s", "remaining_time": "10h 36m 59s"} +{"eps": 0, "objective/kl": 15.29090023, "objective/entropy": -111.93338776, "objective/non_score_reward": -0.76454496, "objective/rlhf_reward": 8.22764301, "objective/scores": 9.0, "policy/approxkl_avg": 0.00121098, "policy/clipfrac_avg": 0.01471991, "loss/policy_avg": 0.09475674, "loss/value_avg": 1.41818285, "val/clipfrac_avg": 0.23112056, "policy/entropy_avg": 0.21949911, "val/ratio": 0.9894408, "val/ratio_var": 0.00019307, "val/num_eos_tokens": 202, "lr": 8.86e-06, "episode": 5136, "epoch": 0.25939394, "global_step/max_steps": "321/1238", "percentage": "25.93%", "elapsed_time": "3h 42m 55s", "remaining_time": "10h 36m 50s"} +{"eps": 0, "objective/kl": 20.26530457, "objective/entropy": -67.4272995, "objective/non_score_reward": -1.01326525, "objective/rlhf_reward": 9.89689064, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0010008, "policy/clipfrac_avg": 0.01303735, "loss/policy_avg": -0.03570066, "loss/value_avg": 0.69536602, "val/clipfrac_avg": 0.21500723, "policy/entropy_avg": 0.15838981, "val/ratio": 0.99003291, "val/ratio_var": 4.196e-05, "val/num_eos_tokens": 316, "lr": 8.85e-06, "episode": 5152, "epoch": 0.26020202, "global_step/max_steps": "322/1238", "percentage": "26.01%", "elapsed_time": "3h 43m 35s", "remaining_time": "10h 36m 4s"} +{"eps": 0, "objective/kl": 20.74421692, "objective/entropy": -34.66552353, "objective/non_score_reward": -1.0372107, "objective/rlhf_reward": 8.30263233, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00107305, "policy/clipfrac_avg": 0.00638404, "loss/policy_avg": -0.10239147, "loss/value_avg": 0.11446619, "val/clipfrac_avg": 0.0446986, "policy/entropy_avg": 0.17284188, "val/ratio": 0.98839307, "val/ratio_var": 3.372e-05, "val/num_eos_tokens": 82, "lr": 8.84e-06, "episode": 5168, "epoch": 0.2610101, "global_step/max_steps": "323/1238", "percentage": "26.09%", "elapsed_time": "3h 44m 17s", "remaining_time": "10h 35m 23s"} +{"eps": 0, "objective/kl": 21.24190521, "objective/entropy": -215.53665161, "objective/non_score_reward": -1.0620954, "objective/rlhf_reward": 8.55118561, "objective/scores": 9.625, "policy/approxkl_avg": 0.00084016, "policy/clipfrac_avg": 0.00653784, "loss/policy_avg": -0.12644696, "loss/value_avg": 0.28114426, "val/clipfrac_avg": 0.04681334, "policy/entropy_avg": 0.1529884, "val/ratio": 0.99068487, "val/ratio_var": 3.289e-05, "val/num_eos_tokens": 128, "lr": 8.83e-06, "episode": 5184, "epoch": 0.26181818, "global_step/max_steps": "324/1238", "percentage": "26.17%", "elapsed_time": "3h 45m 59s", "remaining_time": "10h 37m 31s"} +{"eps": 0, "objective/kl": 17.67336655, "objective/entropy": -130.52597046, "objective/non_score_reward": -0.8836683, "objective/rlhf_reward": 8.81164455, "objective/scores": 9.6875, "policy/approxkl_avg": 0.00126804, "policy/clipfrac_avg": 0.00788056, "loss/policy_avg": 0.32401523, "loss/value_avg": 0.60052997, "val/clipfrac_avg": 0.19150397, "policy/entropy_avg": 0.14477348, "val/ratio": 0.99059284, "val/ratio_var": 5.436e-05, "val/num_eos_tokens": 618, "lr": 8.82e-06, "episode": 5200, "epoch": 0.26262626, "global_step/max_steps": "325/1238", "percentage": "26.25%", "elapsed_time": "3h 46m 37s", "remaining_time": "10h 36m 38s"} +{"eps": 0, "objective/kl": 18.46453667, "objective/entropy": -127.53799438, "objective/non_score_reward": -0.92322683, "objective/rlhf_reward": 7.91271067, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00080325, "policy/clipfrac_avg": 0.00631809, "loss/policy_avg": -0.15733838, "loss/value_avg": 0.50873929, "val/clipfrac_avg": 0.08119471, "policy/entropy_avg": 0.14925909, "val/ratio": 0.99002469, "val/ratio_var": 4.682e-05, "val/num_eos_tokens": 386, "lr": 8.82e-06, "episode": 5216, "epoch": 0.26343434, "global_step/max_steps": "326/1238", "percentage": "26.33%", "elapsed_time": "3h 47m 17s", "remaining_time": "10h 35m 51s"} +{"eps": 0, "objective/kl": 20.63101196, "objective/entropy": -80.75683594, "objective/non_score_reward": -1.03155053, "objective/rlhf_reward": 8.79266834, "objective/scores": 9.8125, "policy/approxkl_avg": 0.00085356, "policy/clipfrac_avg": 0.02039714, "loss/policy_avg": -0.4491275, "loss/value_avg": 0.65885425, "val/clipfrac_avg": 0.06285602, "policy/entropy_avg": 0.14399004, "val/ratio": 0.99045801, "val/ratio_var": 3.422e-05, "val/num_eos_tokens": 202, "lr": 8.81e-06, "episode": 5232, "epoch": 0.26424242, "global_step/max_steps": "327/1238", "percentage": "26.41%", "elapsed_time": "3h 48m 0s", "remaining_time": "10h 35m 13s"} +{"eps": 0, "objective/kl": 20.59920692, "objective/entropy": -80.94671631, "objective/non_score_reward": -1.02996039, "objective/rlhf_reward": 9.35285187, "objective/scores": 10.375, "policy/approxkl_avg": 0.00099675, "policy/clipfrac_avg": 0.02302784, "loss/policy_avg": -0.34798813, "loss/value_avg": 1.16244626, "val/clipfrac_avg": 0.16796532, "policy/entropy_avg": 0.15660107, "val/ratio": 0.9897424, "val/ratio_var": 6.024e-05, "val/num_eos_tokens": 565, "lr": 8.8e-06, "episode": 5248, "epoch": 0.26505051, "global_step/max_steps": "328/1238", "percentage": "26.49%", "elapsed_time": "3h 48m 39s", "remaining_time": "10h 34m 22s"} +{"eps": 0, "objective/kl": 20.20224762, "objective/entropy": -106.59388733, "objective/non_score_reward": -1.0101124, "objective/rlhf_reward": 10.06410599, "objective/scores": 11.0625, "policy/approxkl_avg": 0.00088922, "policy/clipfrac_avg": 0.0140229, "loss/policy_avg": -0.02400905, "loss/value_avg": 1.36432672, "val/clipfrac_avg": 0.11122452, "policy/entropy_avg": 0.15981293, "val/ratio": 0.98904848, "val/ratio_var": 6.913e-05, "val/num_eos_tokens": 46, "lr": 8.79e-06, "episode": 5264, "epoch": 0.26585859, "global_step/max_steps": "329/1238", "percentage": "26.58%", "elapsed_time": "3h 49m 21s", "remaining_time": "10h 33m 42s"} +{"eps": 0, "objective/kl": 17.81369972, "objective/entropy": -62.44646835, "objective/non_score_reward": -0.89068496, "objective/rlhf_reward": 8.99603367, "objective/scores": 9.875, "policy/approxkl_avg": 0.00069007, "policy/clipfrac_avg": 0.00485661, "loss/policy_avg": -0.06329865, "loss/value_avg": 0.54716355, "val/clipfrac_avg": 0.15393913, "policy/entropy_avg": 0.14375496, "val/ratio": 0.99156821, "val/ratio_var": 2.577e-05, "val/num_eos_tokens": 7, "lr": 8.78e-06, "episode": 5280, "epoch": 0.26666667, "global_step/max_steps": "330/1238", "percentage": "26.66%", "elapsed_time": "3h 49m 47s", "remaining_time": "10h 32m 17s"} +{"eps": 0, "objective/kl": 19.03775406, "objective/entropy": -88.05587769, "objective/non_score_reward": -0.95188773, "objective/rlhf_reward": 6.60084677, "objective/scores": 7.5625, "policy/approxkl_avg": 0.00082587, "policy/clipfrac_avg": 0.04584531, "loss/policy_avg": 0.66824538, "loss/value_avg": 2.51472068, "val/clipfrac_avg": 0.24742356, "policy/entropy_avg": 0.12726307, "val/ratio": 0.99084389, "val/ratio_var": 3.244e-05, "val/num_eos_tokens": 210, "lr": 8.77e-06, "episode": 5296, "epoch": 0.26747475, "global_step/max_steps": "331/1238", "percentage": "26.74%", "elapsed_time": "3h 50m 22s", "remaining_time": "10h 31m 15s"} +{"eps": 0, "objective/kl": 16.86630249, "objective/entropy": -166.76419067, "objective/non_score_reward": -0.84331506, "objective/rlhf_reward": 9.95355988, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00083317, "policy/clipfrac_avg": 0.01462307, "loss/policy_avg": -0.07922034, "loss/value_avg": 0.5227114, "val/clipfrac_avg": 0.21186738, "policy/entropy_avg": 0.13784933, "val/ratio": 0.99174094, "val/ratio_var": 4.541e-05, "val/num_eos_tokens": 293, "lr": 8.76e-06, "episode": 5312, "epoch": 0.26828283, "global_step/max_steps": "332/1238", "percentage": "26.82%", "elapsed_time": "3h 51m 15s", "remaining_time": "10h 31m 5s"} +{"eps": 0, "objective/kl": 23.77109528, "objective/entropy": -113.41132355, "objective/non_score_reward": -1.18855476, "objective/rlhf_reward": 8.97160149, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00102982, "policy/clipfrac_avg": 0.00896092, "loss/policy_avg": -0.1037968, "loss/value_avg": 0.22473346, "val/clipfrac_avg": 0.07536854, "policy/entropy_avg": 0.15825367, "val/ratio": 0.98863983, "val/ratio_var": 8.408e-05, "val/num_eos_tokens": 622, "lr": 8.75e-06, "episode": 5328, "epoch": 0.26909091, "global_step/max_steps": "333/1238", "percentage": "26.90%", "elapsed_time": "3h 52m 9s", "remaining_time": "10h 30m 55s"} +{"eps": 0, "objective/kl": 12.80009174, "objective/entropy": -110.30827332, "objective/non_score_reward": -0.64000458, "objective/rlhf_reward": 8.45374489, "objective/scores": 9.125, "policy/approxkl_avg": 0.00067191, "policy/clipfrac_avg": 0.00986943, "loss/policy_avg": 0.2412255, "loss/value_avg": 0.55946636, "val/clipfrac_avg": 0.25742474, "policy/entropy_avg": 0.10278797, "val/ratio": 0.99386144, "val/ratio_var": 4.642e-05, "val/num_eos_tokens": 260, "lr": 8.75e-06, "episode": 5344, "epoch": 0.26989899, "global_step/max_steps": "334/1238", "percentage": "26.98%", "elapsed_time": "3h 52m 44s", "remaining_time": "10h 29m 54s"} +{"eps": 0, "objective/kl": 20.53395271, "objective/entropy": -132.78942871, "objective/non_score_reward": -1.02669764, "objective/rlhf_reward": 8.50845909, "objective/scores": 9.5625, "policy/approxkl_avg": 0.00075486, "policy/clipfrac_avg": 0.00718074, "loss/policy_avg": -0.02042115, "loss/value_avg": 0.58131278, "val/clipfrac_avg": 0.12942752, "policy/entropy_avg": 0.1455729, "val/ratio": 0.99154305, "val/ratio_var": 3.248e-05, "val/num_eos_tokens": 910, "lr": 8.74e-06, "episode": 5360, "epoch": 0.27070707, "global_step/max_steps": "335/1238", "percentage": "27.06%", "elapsed_time": "3h 53m 35s", "remaining_time": "10h 29m 39s"} +{"eps": 0, "objective/kl": 20.93308449, "objective/entropy": -78.7798233, "objective/non_score_reward": -1.04665422, "objective/rlhf_reward": 9.94944, "objective/scores": 11.0, "policy/approxkl_avg": 0.00105658, "policy/clipfrac_avg": 0.01210195, "loss/policy_avg": -0.26860824, "loss/value_avg": 1.66569138, "val/clipfrac_avg": 0.12851554, "policy/entropy_avg": 0.15659451, "val/ratio": 0.99060464, "val/ratio_var": 3.887e-05, "val/num_eos_tokens": 198, "lr": 8.73e-06, "episode": 5376, "epoch": 0.27151515, "global_step/max_steps": "336/1238", "percentage": "27.14%", "elapsed_time": "3h 54m 11s", "remaining_time": "10h 28m 40s"} +{"eps": 0, "objective/kl": 16.70428848, "objective/entropy": -131.01554871, "objective/non_score_reward": -0.83521438, "objective/rlhf_reward": 10.22337914, "objective/scores": 11.0625, "policy/approxkl_avg": 0.0006118, "policy/clipfrac_avg": 0.00482571, "loss/policy_avg": 0.2404552, "loss/value_avg": 1.61451077, "val/clipfrac_avg": 0.15292317, "policy/entropy_avg": 0.10847282, "val/ratio": 0.99263608, "val/ratio_var": 4.497e-05, "val/num_eos_tokens": 780, "lr": 8.72e-06, "episode": 5392, "epoch": 0.27232323, "global_step/max_steps": "337/1238", "percentage": "27.22%", "elapsed_time": "3h 54m 55s", "remaining_time": "10h 28m 6s"} +{"eps": 0, "objective/kl": 18.50728989, "objective/entropy": -91.27498627, "objective/non_score_reward": -0.92536455, "objective/rlhf_reward": 10.20744896, "objective/scores": 11.125, "policy/approxkl_avg": 0.00084911, "policy/clipfrac_avg": 0.00729607, "loss/policy_avg": 0.18228552, "loss/value_avg": 2.50271368, "val/clipfrac_avg": 0.16338566, "policy/entropy_avg": 0.14929378, "val/ratio": 0.98999977, "val/ratio_var": 5.116e-05, "val/num_eos_tokens": 195, "lr": 8.71e-06, "episode": 5408, "epoch": 0.27313131, "global_step/max_steps": "338/1238", "percentage": "27.30%", "elapsed_time": "3h 55m 29s", "remaining_time": "10h 27m 3s"} +{"eps": 0, "objective/kl": 20.30476761, "objective/entropy": -85.34510803, "objective/non_score_reward": -1.01523829, "objective/rlhf_reward": 10.49257469, "objective/scores": 11.5, "policy/approxkl_avg": 0.00090898, "policy/clipfrac_avg": 0.01157038, "loss/policy_avg": 0.10175849, "loss/value_avg": 1.11903191, "val/clipfrac_avg": 0.14293805, "policy/entropy_avg": 0.1657362, "val/ratio": 0.98898011, "val/ratio_var": 5.047e-05, "val/num_eos_tokens": 772, "lr": 8.7e-06, "episode": 5424, "epoch": 0.27393939, "global_step/max_steps": "339/1238", "percentage": "27.38%", "elapsed_time": "3h 56m 10s", "remaining_time": "10h 26m 18s"} +{"eps": 0, "objective/kl": 19.25902176, "objective/entropy": -81.4855423, "objective/non_score_reward": -0.96295106, "objective/rlhf_reward": 9.72454834, "objective/scores": 10.6875, "policy/approxkl_avg": 0.00085995, "policy/clipfrac_avg": 0.01357247, "loss/policy_avg": 0.14838971, "loss/value_avg": 4.2349987, "val/clipfrac_avg": 0.09922881, "policy/entropy_avg": 0.12827253, "val/ratio": 0.99143016, "val/ratio_var": 4.035e-05, "val/num_eos_tokens": 42, "lr": 8.69e-06, "episode": 5440, "epoch": 0.27474747, "global_step/max_steps": "340/1238", "percentage": "27.46%", "elapsed_time": "3h 56m 44s", "remaining_time": "10h 25m 17s"} +{"eps": 0, "objective/kl": 23.56014633, "objective/entropy": -45.80091858, "objective/non_score_reward": -1.17800736, "objective/rlhf_reward": 9.84933662, "objective/scores": 11.0, "policy/approxkl_avg": 0.00107383, "policy/clipfrac_avg": 0.00759343, "loss/policy_avg": -0.46411806, "loss/value_avg": 0.89382219, "val/clipfrac_avg": 0.23792031, "policy/entropy_avg": 0.1959877, "val/ratio": 0.98573726, "val/ratio_var": 0.0001074, "val/num_eos_tokens": 90, "lr": 8.68e-06, "episode": 5456, "epoch": 0.27555556, "global_step/max_steps": "341/1238", "percentage": "27.54%", "elapsed_time": "3h 57m 18s", "remaining_time": "10h 24m 14s"} +{"eps": 0, "objective/kl": 18.86438942, "objective/entropy": -99.86138153, "objective/non_score_reward": -0.94321954, "objective/rlhf_reward": 9.58802986, "objective/scores": 10.5, "policy/approxkl_avg": 0.00063694, "policy/clipfrac_avg": 0.00498361, "loss/policy_avg": -0.70335549, "loss/value_avg": 1.00222385, "val/clipfrac_avg": 0.08656238, "policy/entropy_avg": 0.11473152, "val/ratio": 0.99279153, "val/ratio_var": 2.163e-05, "val/num_eos_tokens": 62, "lr": 8.67e-06, "episode": 5472, "epoch": 0.27636364, "global_step/max_steps": "342/1238", "percentage": "27.63%", "elapsed_time": "3h 58m 4s", "remaining_time": "10h 23m 44s"} +{"eps": 0, "objective/kl": 15.84041691, "objective/entropy": -147.96414185, "objective/non_score_reward": -0.79202092, "objective/rlhf_reward": 11.2236042, "objective/scores": 12.0, "policy/approxkl_avg": 0.00186634, "policy/clipfrac_avg": 0.01593326, "loss/policy_avg": 0.16984278, "loss/value_avg": 1.01955032, "val/clipfrac_avg": 0.16014034, "policy/entropy_avg": 0.12890291, "val/ratio": 0.99203169, "val/ratio_var": 3.812e-05, "val/num_eos_tokens": 729, "lr": 8.67e-06, "episode": 5488, "epoch": 0.27717172, "global_step/max_steps": "343/1238", "percentage": "27.71%", "elapsed_time": "3h 58m 43s", "remaining_time": "10h 22m 53s"} +{"eps": 0, "objective/kl": 16.89540672, "objective/entropy": -118.87034607, "objective/non_score_reward": -0.84477043, "objective/rlhf_reward": 10.32710457, "objective/scores": 11.1875, "policy/approxkl_avg": 0.0007699, "policy/clipfrac_avg": 0.0117946, "loss/policy_avg": 0.05566444, "loss/value_avg": 0.40116769, "val/clipfrac_avg": 0.25469756, "policy/entropy_avg": 0.14203107, "val/ratio": 0.99060631, "val/ratio_var": 5.07e-05, "val/num_eos_tokens": 453, "lr": 8.66e-06, "episode": 5504, "epoch": 0.2779798, "global_step/max_steps": "344/1238", "percentage": "27.79%", "elapsed_time": "3h 59m 19s", "remaining_time": "10h 21m 58s"} +{"eps": 0, "objective/kl": 16.24144554, "objective/entropy": -76.36804199, "objective/non_score_reward": -0.81207234, "objective/rlhf_reward": 11.08636475, "objective/scores": 11.875, "policy/approxkl_avg": 0.00072581, "policy/clipfrac_avg": 0.00532037, "loss/policy_avg": -0.27140141, "loss/value_avg": 1.51310205, "val/clipfrac_avg": 0.14076993, "policy/entropy_avg": 0.12322688, "val/ratio": 0.99152195, "val/ratio_var": 7.229e-05, "val/num_eos_tokens": 9, "lr": 8.65e-06, "episode": 5520, "epoch": 0.27878788, "global_step/max_steps": "345/1238", "percentage": "27.87%", "elapsed_time": "3h 59m 55s", "remaining_time": "10h 21m 1s"} +{"eps": 0, "objective/kl": 23.16811371, "objective/entropy": -73.7339859, "objective/non_score_reward": -1.15840566, "objective/rlhf_reward": 9.6540947, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00107584, "policy/clipfrac_avg": 0.01189534, "loss/policy_avg": 0.31329307, "loss/value_avg": 0.53719366, "val/clipfrac_avg": 0.12817031, "policy/entropy_avg": 0.12182665, "val/ratio": 0.99057567, "val/ratio_var": 3.309e-05, "val/num_eos_tokens": 331, "lr": 8.64e-06, "episode": 5536, "epoch": 0.27959596, "global_step/max_steps": "346/1238", "percentage": "27.95%", "elapsed_time": "4h 0m 24s", "remaining_time": "10h 19m 47s"} +{"eps": 0, "objective/kl": 21.34497452, "objective/entropy": -90.95281219, "objective/non_score_reward": -1.06724882, "objective/rlhf_reward": 11.0577507, "objective/scores": 12.125, "policy/approxkl_avg": 0.00067778, "policy/clipfrac_avg": 0.00522307, "loss/policy_avg": 0.18582414, "loss/value_avg": 0.88265347, "val/clipfrac_avg": 0.27874506, "policy/entropy_avg": 0.12928391, "val/ratio": 0.99114841, "val/ratio_var": 5.034e-05, "val/num_eos_tokens": 424, "lr": 8.63e-06, "episode": 5552, "epoch": 0.28040404, "global_step/max_steps": "347/1238", "percentage": "28.03%", "elapsed_time": "4h 0m 59s", "remaining_time": "10h 18m 48s"} +{"eps": 0, "objective/kl": 24.18220711, "objective/entropy": -148.79641724, "objective/non_score_reward": -1.20911026, "objective/rlhf_reward": 6.92370224, "objective/scores": 8.125, "policy/approxkl_avg": 0.00071714, "policy/clipfrac_avg": 0.00959454, "loss/policy_avg": 0.35247558, "loss/value_avg": 0.57087755, "val/clipfrac_avg": 0.23543015, "policy/entropy_avg": 0.12054634, "val/ratio": 0.99154758, "val/ratio_var": 3.163e-05, "val/num_eos_tokens": 537, "lr": 8.62e-06, "episode": 5568, "epoch": 0.28121212, "global_step/max_steps": "348/1238", "percentage": "28.11%", "elapsed_time": "4h 1m 47s", "remaining_time": "10h 18m 21s"} +{"eps": 0, "objective/kl": 25.37128067, "objective/entropy": -47.9227562, "objective/non_score_reward": -1.26856399, "objective/rlhf_reward": 9.65721798, "objective/scores": 10.9375, "policy/approxkl_avg": 0.00077008, "policy/clipfrac_avg": 0.00470364, "loss/policy_avg": -0.01423438, "loss/value_avg": 0.78442651, "val/clipfrac_avg": 0.26018628, "policy/entropy_avg": 0.14096069, "val/ratio": 0.99032331, "val/ratio_var": 3.461e-05, "val/num_eos_tokens": 25, "lr": 8.61e-06, "episode": 5584, "epoch": 0.2820202, "global_step/max_steps": "349/1238", "percentage": "28.19%", "elapsed_time": "4h 2m 23s", "remaining_time": "10h 17m 25s"} +{"eps": 0, "objective/kl": 25.33921051, "objective/entropy": -116.39802551, "objective/non_score_reward": -1.26696062, "objective/rlhf_reward": 8.07288361, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00099138, "policy/clipfrac_avg": 0.00637183, "loss/policy_avg": -0.09183628, "loss/value_avg": 0.18937214, "val/clipfrac_avg": 0.04358673, "policy/entropy_avg": 0.14559484, "val/ratio": 0.99001849, "val/ratio_var": 3.289e-05, "val/num_eos_tokens": 592, "lr": 8.6e-06, "episode": 5600, "epoch": 0.28282828, "global_step/max_steps": "350/1238", "percentage": "28.27%", "elapsed_time": "4h 3m 9s", "remaining_time": "10h 16m 55s"} +{"eps": 0, "objective/kl": 25.08323288, "objective/entropy": -138.04812622, "objective/non_score_reward": -1.2541616, "objective/rlhf_reward": 7.92161942, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00084973, "policy/clipfrac_avg": 0.01730759, "loss/policy_avg": -0.00683099, "loss/value_avg": 0.9599238, "val/clipfrac_avg": 0.09047566, "policy/entropy_avg": 0.14270031, "val/ratio": 0.99117756, "val/ratio_var": 3.715e-05, "val/num_eos_tokens": 64, "lr": 8.59e-06, "episode": 5616, "epoch": 0.28363636, "global_step/max_steps": "351/1238", "percentage": "28.35%", "elapsed_time": "4h 4m 17s", "remaining_time": "10h 17m 21s"} +{"eps": 0, "objective/kl": 20.58731461, "objective/entropy": -89.55154419, "objective/non_score_reward": -1.02936578, "objective/rlhf_reward": 9.50188446, "objective/scores": 10.5, "policy/approxkl_avg": 0.00413207, "policy/clipfrac_avg": 0.02699372, "loss/policy_avg": -0.29971743, "loss/value_avg": 1.09532559, "val/clipfrac_avg": 0.18198784, "policy/entropy_avg": 0.14103031, "val/ratio": 0.99240708, "val/ratio_var": 0.00060878, "val/num_eos_tokens": 105, "lr": 8.58e-06, "episode": 5632, "epoch": 0.28444444, "global_step/max_steps": "352/1238", "percentage": "28.43%", "elapsed_time": "4h 4m 50s", "remaining_time": "10h 16m 15s"} +{"eps": 0, "objective/kl": 19.54458618, "objective/entropy": -96.51125336, "objective/non_score_reward": -0.9772293, "objective/rlhf_reward": 10.46027088, "objective/scores": 11.4375, "policy/approxkl_avg": 0.00077471, "policy/clipfrac_avg": 0.00631937, "loss/policy_avg": 0.03091885, "loss/value_avg": 0.21527605, "val/clipfrac_avg": 0.06936304, "policy/entropy_avg": 0.12494707, "val/ratio": 0.99165356, "val/ratio_var": 4.327e-05, "val/num_eos_tokens": 302, "lr": 8.57e-06, "episode": 5648, "epoch": 0.28525253, "global_step/max_steps": "353/1238", "percentage": "28.51%", "elapsed_time": "4h 5m 23s", "remaining_time": "10h 15m 13s"} +{"eps": 0, "objective/kl": 19.48657227, "objective/entropy": -39.9475174, "objective/non_score_reward": -0.97432864, "objective/rlhf_reward": 8.47098351, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00126163, "policy/clipfrac_avg": 0.01107365, "loss/policy_avg": 0.08415428, "loss/value_avg": 0.76434475, "val/clipfrac_avg": 0.14175767, "policy/entropy_avg": 0.14904785, "val/ratio": 0.99004298, "val/ratio_var": 5.867e-05, "val/num_eos_tokens": 66, "lr": 8.56e-06, "episode": 5664, "epoch": 0.28606061, "global_step/max_steps": "354/1238", "percentage": "28.59%", "elapsed_time": "4h 5m 58s", "remaining_time": "10h 14m 13s"} +{"eps": 0, "objective/kl": 21.03156853, "objective/entropy": -91.80198669, "objective/non_score_reward": -1.05157852, "objective/rlhf_reward": 8.64764023, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0007716, "policy/clipfrac_avg": 0.00685991, "loss/policy_avg": -0.00933796, "loss/value_avg": 0.32016802, "val/clipfrac_avg": 0.10722832, "policy/entropy_avg": 0.15067577, "val/ratio": 0.99011457, "val/ratio_var": 4.588e-05, "val/num_eos_tokens": 556, "lr": 8.55e-06, "episode": 5680, "epoch": 0.28686869, "global_step/max_steps": "355/1238", "percentage": "28.68%", "elapsed_time": "4h 6m 32s", "remaining_time": "10h 13m 12s"} +{"eps": 0, "objective/kl": 26.18867302, "objective/entropy": -147.03330994, "objective/non_score_reward": -1.3094337, "objective/rlhf_reward": 8.80189419, "objective/scores": 10.125, "policy/approxkl_avg": 0.00080816, "policy/clipfrac_avg": 0.00388069, "loss/policy_avg": -0.44288927, "loss/value_avg": 0.80144405, "val/clipfrac_avg": 0.1697432, "policy/entropy_avg": 0.15047848, "val/ratio": 0.98932225, "val/ratio_var": 5.329e-05, "val/num_eos_tokens": 353, "lr": 8.54e-06, "episode": 5696, "epoch": 0.28767677, "global_step/max_steps": "356/1238", "percentage": "28.76%", "elapsed_time": "4h 8m 14s", "remaining_time": "10h 15m 0s"} +{"eps": 0, "objective/kl": 20.14461899, "objective/entropy": -75.95680237, "objective/non_score_reward": -1.007231, "objective/rlhf_reward": 9.78964424, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00101262, "policy/clipfrac_avg": 0.00791793, "loss/policy_avg": -0.06810799, "loss/value_avg": 0.42201668, "val/clipfrac_avg": 0.06293391, "policy/entropy_avg": 0.15483475, "val/ratio": 0.99017513, "val/ratio_var": 6.756e-05, "val/num_eos_tokens": 70, "lr": 8.54e-06, "episode": 5712, "epoch": 0.28848485, "global_step/max_steps": "357/1238", "percentage": "28.84%", "elapsed_time": "4h 8m 45s", "remaining_time": "10h 13m 53s"} +{"eps": 0, "objective/kl": 17.09016228, "objective/entropy": -78.61557007, "objective/non_score_reward": -0.85450816, "objective/rlhf_reward": 10.02439785, "objective/scores": 10.875, "policy/approxkl_avg": 0.00084895, "policy/clipfrac_avg": 0.00721886, "loss/policy_avg": 0.13574761, "loss/value_avg": 1.21861565, "val/clipfrac_avg": 0.13289574, "policy/entropy_avg": 0.14081085, "val/ratio": 0.99113947, "val/ratio_var": 5.176e-05, "val/num_eos_tokens": 77, "lr": 8.53e-06, "episode": 5728, "epoch": 0.28929293, "global_step/max_steps": "358/1238", "percentage": "28.92%", "elapsed_time": "4h 9m 19s", "remaining_time": "10h 12m 52s"} +{"eps": 0, "objective/kl": 18.76550293, "objective/entropy": -91.85313416, "objective/non_score_reward": -0.93827516, "objective/rlhf_reward": 7.57734966, "objective/scores": 8.5, "policy/approxkl_avg": 0.0012549, "policy/clipfrac_avg": 0.01372809, "loss/policy_avg": 0.85278058, "loss/value_avg": 1.68280113, "val/clipfrac_avg": 0.409352, "policy/entropy_avg": 0.1779511, "val/ratio": 0.99023509, "val/ratio_var": 4.967e-05, "val/num_eos_tokens": 160, "lr": 8.52e-06, "episode": 5744, "epoch": 0.29010101, "global_step/max_steps": "359/1238", "percentage": "29.00%", "elapsed_time": "4h 9m 46s", "remaining_time": "10h 11m 34s"} +{"eps": 0, "objective/kl": 19.0248909, "objective/entropy": -113.52433014, "objective/non_score_reward": -0.95124453, "objective/rlhf_reward": 10.0448494, "objective/scores": 11.0, "policy/approxkl_avg": 0.00105592, "policy/clipfrac_avg": 0.01111959, "loss/policy_avg": 0.38345039, "loss/value_avg": 1.26634002, "val/clipfrac_avg": 0.20008543, "policy/entropy_avg": 0.15532589, "val/ratio": 0.99064589, "val/ratio_var": 5.072e-05, "val/num_eos_tokens": 110, "lr": 8.51e-06, "episode": 5760, "epoch": 0.29090909, "global_step/max_steps": "360/1238", "percentage": "29.08%", "elapsed_time": "4h 10m 21s", "remaining_time": "10h 10m 35s"} +{"eps": 0, "objective/kl": 22.22720718, "objective/entropy": -220.52227783, "objective/non_score_reward": -1.11136031, "objective/rlhf_reward": 8.0722332, "objective/scores": 9.1875, "policy/approxkl_avg": 0.00094325, "policy/clipfrac_avg": 0.01834722, "loss/policy_avg": 0.07996695, "loss/value_avg": 0.75556529, "val/clipfrac_avg": 0.11736906, "policy/entropy_avg": 0.15595913, "val/ratio": 0.9901768, "val/ratio_var": 7.676e-05, "val/num_eos_tokens": 117, "lr": 8.5e-06, "episode": 5776, "epoch": 0.29171717, "global_step/max_steps": "361/1238", "percentage": "29.16%", "elapsed_time": "4h 12m 1s", "remaining_time": "10h 12m 16s"} +{"eps": 0, "objective/kl": 19.32946396, "objective/entropy": -127.60949707, "objective/non_score_reward": -0.96647316, "objective/rlhf_reward": 8.40852737, "objective/scores": 9.375, "policy/approxkl_avg": 0.00060409, "policy/clipfrac_avg": 0.00478483, "loss/policy_avg": 0.24114887, "loss/value_avg": 2.61820555, "val/clipfrac_avg": 0.27273598, "policy/entropy_avg": 0.12860215, "val/ratio": 0.99146682, "val/ratio_var": 5.469e-05, "val/num_eos_tokens": 598, "lr": 8.49e-06, "episode": 5792, "epoch": 0.29252525, "global_step/max_steps": "362/1238", "percentage": "29.24%", "elapsed_time": "4h 12m 40s", "remaining_time": "10h 11m 26s"} +{"eps": 0, "objective/kl": 24.04833603, "objective/entropy": -125.06207275, "objective/non_score_reward": -1.2024169, "objective/rlhf_reward": 8.11398888, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00094437, "policy/clipfrac_avg": 0.01316667, "loss/policy_avg": 0.25809526, "loss/value_avg": 0.47065228, "val/clipfrac_avg": 0.10755911, "policy/entropy_avg": 0.14909744, "val/ratio": 0.99038935, "val/ratio_var": 2.328e-05, "val/num_eos_tokens": 628, "lr": 8.48e-06, "episode": 5808, "epoch": 0.29333333, "global_step/max_steps": "363/1238", "percentage": "29.32%", "elapsed_time": "4h 13m 22s", "remaining_time": "10h 10m 44s"} +{"eps": 0, "objective/kl": 28.9068222, "objective/entropy": -92.45407867, "objective/non_score_reward": -1.44534111, "objective/rlhf_reward": 8.74997139, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00108547, "policy/clipfrac_avg": 0.02740762, "loss/policy_avg": 0.15223712, "loss/value_avg": 1.06443167, "val/clipfrac_avg": 0.1557045, "policy/entropy_avg": 0.1836195, "val/ratio": 0.9883889, "val/ratio_var": 3.198e-05, "val/num_eos_tokens": 6, "lr": 8.47e-06, "episode": 5824, "epoch": 0.29414141, "global_step/max_steps": "364/1238", "percentage": "29.40%", "elapsed_time": "4h 14m 5s", "remaining_time": "10h 10m 4s"} +{"eps": 0, "objective/kl": 27.18421555, "objective/entropy": -123.84210205, "objective/non_score_reward": -1.35921073, "objective/rlhf_reward": 7.44547701, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00088802, "policy/clipfrac_avg": 0.00461779, "loss/policy_avg": 0.00158001, "loss/value_avg": 0.27369469, "val/clipfrac_avg": 0.18546769, "policy/entropy_avg": 0.16141415, "val/ratio": 0.98912162, "val/ratio_var": 4.489e-05, "val/num_eos_tokens": 360, "lr": 8.46e-06, "episode": 5840, "epoch": 0.29494949, "global_step/max_steps": "365/1238", "percentage": "29.48%", "elapsed_time": "4h 15m 3s", "remaining_time": "10h 10m 2s"} +{"eps": 0, "objective/kl": 24.24879456, "objective/entropy": -79.13996887, "objective/non_score_reward": -1.21243978, "objective/rlhf_reward": 7.16060734, "objective/scores": 8.375, "policy/approxkl_avg": 0.00080229, "policy/clipfrac_avg": 0.00607715, "loss/policy_avg": 0.01277543, "loss/value_avg": 0.48028678, "val/clipfrac_avg": 0.04264098, "policy/entropy_avg": 0.16005898, "val/ratio": 0.99026191, "val/ratio_var": 3.515e-05, "val/num_eos_tokens": 176, "lr": 8.45e-06, "episode": 5856, "epoch": 0.29575758, "global_step/max_steps": "366/1238", "percentage": "29.56%", "elapsed_time": "4h 15m 44s", "remaining_time": "10h 9m 18s"} +{"eps": 0, "objective/kl": 32.35683823, "objective/entropy": -266.6590271, "objective/non_score_reward": -1.61784196, "objective/rlhf_reward": 8.08918953, "objective/scores": 9.6875, "policy/approxkl_avg": 0.00062896, "policy/clipfrac_avg": 0.00919522, "loss/policy_avg": -0.18061696, "loss/value_avg": 0.53965986, "val/clipfrac_avg": 0.10509516, "policy/entropy_avg": 0.11503601, "val/ratio": 0.99290228, "val/ratio_var": 3.285e-05, "val/num_eos_tokens": 414, "lr": 8.44e-06, "episode": 5872, "epoch": 0.29656566, "global_step/max_steps": "367/1238", "percentage": "29.64%", "elapsed_time": "4h 17m 33s", "remaining_time": "10h 11m 15s"} +{"eps": 0, "objective/kl": 26.48159027, "objective/entropy": -54.53785324, "objective/non_score_reward": -1.32407951, "objective/rlhf_reward": 11.02748299, "objective/scores": 12.375, "policy/approxkl_avg": 0.00106575, "policy/clipfrac_avg": 0.00677007, "loss/policy_avg": -0.1608128, "loss/value_avg": 1.95534754, "val/clipfrac_avg": 0.19464913, "policy/entropy_avg": 0.18307781, "val/ratio": 0.98815948, "val/ratio_var": 5.794e-05, "val/num_eos_tokens": 329, "lr": 8.43e-06, "episode": 5888, "epoch": 0.29737374, "global_step/max_steps": "368/1238", "percentage": "29.73%", "elapsed_time": "4h 18m 6s", "remaining_time": "10h 10m 13s"} +{"eps": 0, "objective/kl": 27.1149025, "objective/entropy": -171.61798096, "objective/non_score_reward": -1.35574508, "objective/rlhf_reward": 7.88253641, "objective/scores": 9.25, "policy/approxkl_avg": 0.00094881, "policy/clipfrac_avg": 0.00842956, "loss/policy_avg": -0.06483447, "loss/value_avg": 0.38115546, "val/clipfrac_avg": 0.0948478, "policy/entropy_avg": 0.17403793, "val/ratio": 0.98893237, "val/ratio_var": 5.187e-05, "val/num_eos_tokens": 417, "lr": 8.42e-06, "episode": 5904, "epoch": 0.29818182, "global_step/max_steps": "369/1238", "percentage": "29.81%", "elapsed_time": "4h 19m 49s", "remaining_time": "10h 11m 52s"} +{"eps": 0, "objective/kl": 25.17058945, "objective/entropy": -61.57228088, "objective/non_score_reward": -1.25852942, "objective/rlhf_reward": 7.51881456, "objective/scores": 8.75, "policy/approxkl_avg": 0.00117907, "policy/clipfrac_avg": 0.00510487, "loss/policy_avg": -0.01815527, "loss/value_avg": 0.25550494, "val/clipfrac_avg": 0.06022166, "policy/entropy_avg": 0.1540451, "val/ratio": 0.98927152, "val/ratio_var": 3.491e-05, "val/num_eos_tokens": 108, "lr": 8.41e-06, "episode": 5920, "epoch": 0.2989899, "global_step/max_steps": "370/1238", "percentage": "29.89%", "elapsed_time": "4h 21m 27s", "remaining_time": "10h 13m 22s"} +{"eps": 0, "objective/kl": 25.24039841, "objective/entropy": -62.79341125, "objective/non_score_reward": -1.26201987, "objective/rlhf_reward": 9.94891739, "objective/scores": 11.1875, "policy/approxkl_avg": 0.00092683, "policy/clipfrac_avg": 0.02191114, "loss/policy_avg": -0.10499699, "loss/value_avg": 0.62814313, "val/clipfrac_avg": 0.10369567, "policy/entropy_avg": 0.15642357, "val/ratio": 0.98944676, "val/ratio_var": 4.366e-05, "val/num_eos_tokens": 82, "lr": 8.4e-06, "episode": 5936, "epoch": 0.29979798, "global_step/max_steps": "371/1238", "percentage": "29.97%", "elapsed_time": "4h 22m 32s", "remaining_time": "10h 13m 31s"} +{"eps": 0, "objective/kl": 24.83453178, "objective/entropy": -90.6949234, "objective/non_score_reward": -1.24172664, "objective/rlhf_reward": 8.82468033, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00130752, "policy/clipfrac_avg": 0.01163518, "loss/policy_avg": -0.37284052, "loss/value_avg": 1.20059097, "val/clipfrac_avg": 0.09476368, "policy/entropy_avg": 0.14268023, "val/ratio": 0.99033725, "val/ratio_var": 5.384e-05, "val/num_eos_tokens": 348, "lr": 8.39e-06, "episode": 5952, "epoch": 0.30060606, "global_step/max_steps": "372/1238", "percentage": "30.05%", "elapsed_time": "4h 23m 32s", "remaining_time": "10h 13m 31s"} +{"eps": 0, "objective/kl": 23.9215889, "objective/entropy": -77.07698822, "objective/non_score_reward": -1.19607949, "objective/rlhf_reward": 8.25704575, "objective/scores": 9.4375, "policy/approxkl_avg": 0.00106542, "policy/clipfrac_avg": 0.01393349, "loss/policy_avg": -0.26838738, "loss/value_avg": 0.81974655, "val/clipfrac_avg": 0.11054794, "policy/entropy_avg": 0.16466379, "val/ratio": 0.99144006, "val/ratio_var": 7.724e-05, "val/num_eos_tokens": 452, "lr": 8.38e-06, "episode": 5968, "epoch": 0.30141414, "global_step/max_steps": "373/1238", "percentage": "30.13%", "elapsed_time": "4h 24m 3s", "remaining_time": "10h 12m 21s"} +{"eps": 0, "objective/kl": 20.35816193, "objective/entropy": -90.79975891, "objective/non_score_reward": -1.0179081, "objective/rlhf_reward": 9.1461544, "objective/scores": 10.1875, "policy/approxkl_avg": 0.00116333, "policy/clipfrac_avg": 0.01132894, "loss/policy_avg": 0.05047921, "loss/value_avg": 0.25139078, "val/clipfrac_avg": 0.09338258, "policy/entropy_avg": 0.15205002, "val/ratio": 0.98993981, "val/ratio_var": 3.125e-05, "val/num_eos_tokens": 292, "lr": 8.37e-06, "episode": 5984, "epoch": 0.30222222, "global_step/max_steps": "374/1238", "percentage": "30.21%", "elapsed_time": "4h 24m 51s", "remaining_time": "10h 11m 52s"} +{"eps": 0, "objective/kl": 19.31946564, "objective/entropy": -152.45593262, "objective/non_score_reward": -0.96597326, "objective/rlhf_reward": 10.71762085, "objective/scores": 11.6875, "policy/approxkl_avg": 0.00062881, "policy/clipfrac_avg": 0.01824055, "loss/policy_avg": -0.24270931, "loss/value_avg": 1.03236771, "val/clipfrac_avg": 0.23011316, "policy/entropy_avg": 0.14447784, "val/ratio": 0.99268675, "val/ratio_var": 3.975e-05, "val/num_eos_tokens": 572, "lr": 8.36e-06, "episode": 6000, "epoch": 0.3030303, "global_step/max_steps": "375/1238", "percentage": "30.29%", "elapsed_time": "4h 25m 45s", "remaining_time": "10h 11m 35s"} +{"eps": 0, "objective/kl": 19.16396332, "objective/entropy": -204.82409668, "objective/non_score_reward": -0.95819819, "objective/rlhf_reward": 10.19023895, "objective/scores": 11.125, "policy/approxkl_avg": 0.0006085, "policy/clipfrac_avg": 0.00505862, "loss/policy_avg": -0.62048066, "loss/value_avg": 0.68571341, "val/clipfrac_avg": 0.15228465, "policy/entropy_avg": 0.1286006, "val/ratio": 0.99278796, "val/ratio_var": 4.832e-05, "val/num_eos_tokens": 235, "lr": 8.35e-06, "episode": 6016, "epoch": 0.30383838, "global_step/max_steps": "376/1238", "percentage": "30.37%", "elapsed_time": "4h 27m 6s", "remaining_time": "10h 12m 21s"} +{"eps": 0, "objective/kl": 20.5356369, "objective/entropy": -81.70239258, "objective/non_score_reward": -1.02678192, "objective/rlhf_reward": 9.44196796, "objective/scores": 10.5, "policy/approxkl_avg": 0.00087467, "policy/clipfrac_avg": 0.00569082, "loss/policy_avg": -0.05810073, "loss/value_avg": 0.49776226, "val/clipfrac_avg": 0.22529854, "policy/entropy_avg": 0.15449715, "val/ratio": 0.98919857, "val/ratio_var": 4.928e-05, "val/num_eos_tokens": 62, "lr": 8.34e-06, "episode": 6032, "epoch": 0.30464646, "global_step/max_steps": "377/1238", "percentage": "30.45%", "elapsed_time": "4h 27m 46s", "remaining_time": "10h 11m 33s"} +{"eps": 0, "objective/kl": 19.32754517, "objective/entropy": -140.73626709, "objective/non_score_reward": -0.96637732, "objective/rlhf_reward": 10.24846649, "objective/scores": 11.1875, "policy/approxkl_avg": 0.00069842, "policy/clipfrac_avg": 0.00471724, "loss/policy_avg": -0.58219928, "loss/value_avg": 1.16544604, "val/clipfrac_avg": 0.29272497, "policy/entropy_avg": 0.12701988, "val/ratio": 0.99226636, "val/ratio_var": 5.017e-05, "val/num_eos_tokens": 51, "lr": 8.33e-06, "episode": 6048, "epoch": 0.30545455, "global_step/max_steps": "378/1238", "percentage": "30.53%", "elapsed_time": "4h 28m 25s", "remaining_time": "10h 10m 42s"} +{"eps": 0, "objective/kl": 20.27896881, "objective/entropy": -77.88591766, "objective/non_score_reward": -1.01394844, "objective/rlhf_reward": 10.25948906, "objective/scores": 11.25, "policy/approxkl_avg": 0.00085488, "policy/clipfrac_avg": 0.00871541, "loss/policy_avg": 0.37417129, "loss/value_avg": 1.05752647, "val/clipfrac_avg": 0.27003729, "policy/entropy_avg": 0.14256516, "val/ratio": 0.99056429, "val/ratio_var": 4.365e-05, "val/num_eos_tokens": 68, "lr": 8.32e-06, "episode": 6064, "epoch": 0.30626263, "global_step/max_steps": "379/1238", "percentage": "30.61%", "elapsed_time": "4h 28m 56s", "remaining_time": "10h 9m 33s"} +{"eps": 0, "objective/kl": 16.52803993, "objective/entropy": -30.36887169, "objective/non_score_reward": -0.82640195, "objective/rlhf_reward": 8.12281609, "objective/scores": 8.9375, "policy/approxkl_avg": 0.00109576, "policy/clipfrac_avg": 0.0085571, "loss/policy_avg": 0.13715182, "loss/value_avg": 0.33893001, "val/clipfrac_avg": 0.09286787, "policy/entropy_avg": 0.14247894, "val/ratio": 0.98989588, "val/ratio_var": 2.362e-05, "val/num_eos_tokens": 117, "lr": 8.31e-06, "episode": 6080, "epoch": 0.30707071, "global_step/max_steps": "380/1238", "percentage": "30.69%", "elapsed_time": "4h 29m 37s", "remaining_time": "10h 8m 47s"} +{"eps": 0, "objective/kl": 23.60455704, "objective/entropy": -30.00856018, "objective/non_score_reward": -1.18022788, "objective/rlhf_reward": 8.14399052, "objective/scores": 9.3125, "policy/approxkl_avg": 0.00103185, "policy/clipfrac_avg": 0.0082549, "loss/policy_avg": 0.09494101, "loss/value_avg": 0.45570412, "val/clipfrac_avg": 0.06098241, "policy/entropy_avg": 0.18322468, "val/ratio": 0.98866338, "val/ratio_var": 3e-05, "val/num_eos_tokens": 592, "lr": 8.3e-06, "episode": 6096, "epoch": 0.30787879, "global_step/max_steps": "381/1238", "percentage": "30.78%", "elapsed_time": "4h 30m 41s", "remaining_time": "10h 8m 53s"} +{"eps": 0, "objective/kl": 21.36620331, "objective/entropy": -116.08049011, "objective/non_score_reward": -1.06831002, "objective/rlhf_reward": 10.90825272, "objective/scores": 12.0, "policy/approxkl_avg": 0.00100328, "policy/clipfrac_avg": 0.00768701, "loss/policy_avg": -0.11349489, "loss/value_avg": 0.73911536, "val/clipfrac_avg": 0.11116229, "policy/entropy_avg": 0.13149738, "val/ratio": 0.99097496, "val/ratio_var": 3.599e-05, "val/num_eos_tokens": 273, "lr": 8.29e-06, "episode": 6112, "epoch": 0.30868687, "global_step/max_steps": "382/1238", "percentage": "30.86%", "elapsed_time": "4h 31m 19s", "remaining_time": "10h 8m 0s"} +{"eps": 0, "objective/kl": 24.36815262, "objective/entropy": -64.46329498, "objective/non_score_reward": -1.21840763, "objective/rlhf_reward": 7.60581112, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00108719, "policy/clipfrac_avg": 0.00945587, "loss/policy_avg": 0.26366717, "loss/value_avg": 0.31152695, "val/clipfrac_avg": 0.17807867, "policy/entropy_avg": 0.20527267, "val/ratio": 0.98895895, "val/ratio_var": 4.264e-05, "val/num_eos_tokens": 14, "lr": 8.28e-06, "episode": 6128, "epoch": 0.30949495, "global_step/max_steps": "383/1238", "percentage": "30.94%", "elapsed_time": "4h 31m 48s", "remaining_time": "10h 6m 45s"} +{"eps": 0, "objective/kl": 23.17105675, "objective/entropy": -112.77015686, "objective/non_score_reward": -1.15855289, "objective/rlhf_reward": 11.26332283, "objective/scores": 12.4375, "policy/approxkl_avg": 0.00091175, "policy/clipfrac_avg": 0.01714495, "loss/policy_avg": -0.25645226, "loss/value_avg": 1.2739346, "val/clipfrac_avg": 0.21119374, "policy/entropy_avg": 0.14481926, "val/ratio": 0.99124002, "val/ratio_var": 3.968e-05, "val/num_eos_tokens": 329, "lr": 8.27e-06, "episode": 6144, "epoch": 0.31030303, "global_step/max_steps": "384/1238", "percentage": "31.02%", "elapsed_time": "4h 32m 25s", "remaining_time": "10h 5m 51s"} +{"eps": 0, "objective/kl": 22.00409317, "objective/entropy": -142.62236023, "objective/non_score_reward": -1.10020471, "objective/rlhf_reward": 8.98573303, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00101803, "policy/clipfrac_avg": 0.00969874, "loss/policy_avg": 0.22879836, "loss/value_avg": 0.65312672, "val/clipfrac_avg": 0.10454543, "policy/entropy_avg": 0.15434915, "val/ratio": 0.99058336, "val/ratio_var": 8.005e-05, "val/num_eos_tokens": 182, "lr": 8.26e-06, "episode": 6160, "epoch": 0.31111111, "global_step/max_steps": "385/1238", "percentage": "31.10%", "elapsed_time": "4h 33m 54s", "remaining_time": "10h 6m 51s"} +{"eps": 0, "objective/kl": 24.10480118, "objective/entropy": -98.07450867, "objective/non_score_reward": -1.20524001, "objective/rlhf_reward": 9.251791, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00085699, "policy/clipfrac_avg": 0.00961955, "loss/policy_avg": 0.19027102, "loss/value_avg": 0.6345529, "val/clipfrac_avg": 0.04851881, "policy/entropy_avg": 0.15359974, "val/ratio": 0.98995084, "val/ratio_var": 4.097e-05, "val/num_eos_tokens": 67, "lr": 8.25e-06, "episode": 6176, "epoch": 0.31191919, "global_step/max_steps": "386/1238", "percentage": "31.18%", "elapsed_time": "4h 34m 31s", "remaining_time": "10h 5m 57s"} +{"eps": 0, "objective/kl": 22.46795654, "objective/entropy": -110.41493988, "objective/non_score_reward": -1.12339783, "objective/rlhf_reward": 8.35316467, "objective/scores": 9.5, "policy/approxkl_avg": 0.00095433, "policy/clipfrac_avg": 0.01460526, "loss/policy_avg": 0.17917794, "loss/value_avg": 0.36408013, "val/clipfrac_avg": 0.16161151, "policy/entropy_avg": 0.16858912, "val/ratio": 0.98856157, "val/ratio_var": 4.822e-05, "val/num_eos_tokens": 273, "lr": 8.24e-06, "episode": 6192, "epoch": 0.31272727, "global_step/max_steps": "387/1238", "percentage": "31.26%", "elapsed_time": "4h 35m 18s", "remaining_time": "10h 5m 23s"} +{"eps": 0, "objective/kl": 19.6662941, "objective/entropy": -77.30952454, "objective/non_score_reward": -0.98331463, "objective/rlhf_reward": 10.78231049, "objective/scores": 11.75, "policy/approxkl_avg": 0.00086054, "policy/clipfrac_avg": 0.01045419, "loss/policy_avg": -0.271819, "loss/value_avg": 1.2867136, "val/clipfrac_avg": 0.25079706, "policy/entropy_avg": 0.16277957, "val/ratio": 0.98961449, "val/ratio_var": 5.392e-05, "val/num_eos_tokens": 135, "lr": 8.23e-06, "episode": 6208, "epoch": 0.31353535, "global_step/max_steps": "388/1238", "percentage": "31.34%", "elapsed_time": "4h 36m 13s", "remaining_time": "10h 5m 8s"} +{"eps": 0, "objective/kl": 24.126091, "objective/entropy": -50.25094986, "objective/non_score_reward": -1.20630455, "objective/rlhf_reward": 8.6647892, "objective/scores": 9.875, "policy/approxkl_avg": 0.00111842, "policy/clipfrac_avg": 0.01398492, "loss/policy_avg": -0.24307014, "loss/value_avg": 0.34959793, "val/clipfrac_avg": 0.06929454, "policy/entropy_avg": 0.19856453, "val/ratio": 0.98886514, "val/ratio_var": 4.259e-05, "val/num_eos_tokens": 188, "lr": 8.22e-06, "episode": 6224, "epoch": 0.31434343, "global_step/max_steps": "389/1238", "percentage": "31.42%", "elapsed_time": "4h 36m 58s", "remaining_time": "10h 4m 30s"} +{"eps": 0, "objective/kl": 21.65674019, "objective/entropy": -112.65766907, "objective/non_score_reward": -1.08283699, "objective/rlhf_reward": 9.6671629, "objective/scores": 10.75, "policy/approxkl_avg": 0.00098673, "policy/clipfrac_avg": 0.00674868, "loss/policy_avg": -0.42672151, "loss/value_avg": 0.93147004, "val/clipfrac_avg": 0.14616436, "policy/entropy_avg": 0.19133377, "val/ratio": 0.98861587, "val/ratio_var": 5.444e-05, "val/num_eos_tokens": 10, "lr": 8.21e-06, "episode": 6240, "epoch": 0.31515152, "global_step/max_steps": "390/1238", "percentage": "31.50%", "elapsed_time": "4h 38m 39s", "remaining_time": "10h 5m 54s"} +{"eps": 0, "objective/kl": 18.75835609, "objective/entropy": -93.58556366, "objective/non_score_reward": -0.93791783, "objective/rlhf_reward": 10.55426979, "objective/scores": 11.5, "policy/approxkl_avg": 0.0006623, "policy/clipfrac_avg": 0.01037301, "loss/policy_avg": -0.27975157, "loss/value_avg": 0.42042723, "val/clipfrac_avg": 0.10715407, "policy/entropy_avg": 0.12451267, "val/ratio": 0.99253738, "val/ratio_var": 3.973e-05, "val/num_eos_tokens": 243, "lr": 8.2e-06, "episode": 6256, "epoch": 0.3159596, "global_step/max_steps": "391/1238", "percentage": "31.58%", "elapsed_time": "4h 39m 18s", "remaining_time": "10h 5m 3s"} +{"eps": 0, "objective/kl": 22.55490112, "objective/entropy": -50.36949921, "objective/non_score_reward": -1.12774515, "objective/rlhf_reward": 10.44256783, "objective/scores": 11.5625, "policy/approxkl_avg": 0.0012944, "policy/clipfrac_avg": 0.01801797, "loss/policy_avg": -0.203664, "loss/value_avg": 0.83599526, "val/clipfrac_avg": 0.15647155, "policy/entropy_avg": 0.17497826, "val/ratio": 0.9895494, "val/ratio_var": 0.00010073, "val/num_eos_tokens": 535, "lr": 8.19e-06, "episode": 6272, "epoch": 0.31676768, "global_step/max_steps": "392/1238", "percentage": "31.66%", "elapsed_time": "4h 39m 49s", "remaining_time": "10h 3m 54s"} +{"eps": 0, "objective/kl": 23.50112152, "objective/entropy": -77.18543243, "objective/non_score_reward": -1.17505598, "objective/rlhf_reward": 9.56713104, "objective/scores": 10.75, "policy/approxkl_avg": 0.00097064, "policy/clipfrac_avg": 0.0163747, "loss/policy_avg": -0.25137085, "loss/value_avg": 0.57025486, "val/clipfrac_avg": 0.16020085, "policy/entropy_avg": 0.16623497, "val/ratio": 0.98955262, "val/ratio_var": 4.694e-05, "val/num_eos_tokens": 31, "lr": 8.18e-06, "episode": 6288, "epoch": 0.31757576, "global_step/max_steps": "393/1238", "percentage": "31.74%", "elapsed_time": "4h 40m 38s", "remaining_time": "10h 3m 24s"} +{"eps": 0, "objective/kl": 24.79729843, "objective/entropy": -76.8727951, "objective/non_score_reward": -1.23986495, "objective/rlhf_reward": 10.15857315, "objective/scores": 11.375, "policy/approxkl_avg": 0.00086678, "policy/clipfrac_avg": 0.00701544, "loss/policy_avg": -0.2903949, "loss/value_avg": 0.71619141, "val/clipfrac_avg": 0.06622799, "policy/entropy_avg": 0.15674305, "val/ratio": 0.98925191, "val/ratio_var": 6.106e-05, "val/num_eos_tokens": 344, "lr": 8.17e-06, "episode": 6304, "epoch": 0.31838384, "global_step/max_steps": "394/1238", "percentage": "31.83%", "elapsed_time": "4h 41m 15s", "remaining_time": "10h 2m 29s"} +{"eps": 0, "objective/kl": 21.72610664, "objective/entropy": -162.07409668, "objective/non_score_reward": -1.08630526, "objective/rlhf_reward": 8.61291313, "objective/scores": 9.6875, "policy/approxkl_avg": 0.00068591, "policy/clipfrac_avg": 0.00427837, "loss/policy_avg": -0.03084909, "loss/value_avg": 0.2917251, "val/clipfrac_avg": 0.08182407, "policy/entropy_avg": 0.13499069, "val/ratio": 0.99129379, "val/ratio_var": 2.832e-05, "val/num_eos_tokens": 190, "lr": 8.16e-06, "episode": 6320, "epoch": 0.31919192, "global_step/max_steps": "395/1238", "percentage": "31.91%", "elapsed_time": "4h 42m 56s", "remaining_time": "10h 3m 49s"} +{"eps": 0, "objective/kl": 19.76261711, "objective/entropy": -90.22354126, "objective/non_score_reward": -0.98813087, "objective/rlhf_reward": 10.91811943, "objective/scores": 11.875, "policy/approxkl_avg": 0.0008516, "policy/clipfrac_avg": 0.00823319, "loss/policy_avg": -0.135141, "loss/value_avg": 2.26446295, "val/clipfrac_avg": 0.27357757, "policy/entropy_avg": 0.14680672, "val/ratio": 0.99008638, "val/ratio_var": 6.002e-05, "val/num_eos_tokens": 119, "lr": 8.15e-06, "episode": 6336, "epoch": 0.32, "global_step/max_steps": "396/1238", "percentage": "31.99%", "elapsed_time": "4h 43m 33s", "remaining_time": "10h 2m 54s"} +{"eps": 0, "objective/kl": 21.18748665, "objective/entropy": -150.10548401, "objective/non_score_reward": -1.05937433, "objective/rlhf_reward": 10.34687614, "objective/scores": 11.375, "policy/approxkl_avg": 0.00079274, "policy/clipfrac_avg": 0.01744936, "loss/policy_avg": -0.52096564, "loss/value_avg": 0.57558888, "val/clipfrac_avg": 0.11896036, "policy/entropy_avg": 0.14124477, "val/ratio": 0.99147928, "val/ratio_var": 5.131e-05, "val/num_eos_tokens": 527, "lr": 8.14e-06, "episode": 6352, "epoch": 0.32080808, "global_step/max_steps": "397/1238", "percentage": "32.07%", "elapsed_time": "4h 44m 8s", "remaining_time": "10h 1m 54s"} +{"eps": 0, "objective/kl": 22.37616348, "objective/entropy": -149.72897339, "objective/non_score_reward": -1.11880815, "objective/rlhf_reward": 10.24056721, "objective/scores": 11.375, "policy/approxkl_avg": 0.0007364, "policy/clipfrac_avg": 0.00870537, "loss/policy_avg": -0.4057698, "loss/value_avg": 0.33822232, "val/clipfrac_avg": 0.09983382, "policy/entropy_avg": 0.13206863, "val/ratio": 0.9914937, "val/ratio_var": 3.097e-05, "val/num_eos_tokens": 92, "lr": 8.13e-06, "episode": 6368, "epoch": 0.32161616, "global_step/max_steps": "398/1238", "percentage": "32.15%", "elapsed_time": "4h 45m 5s", "remaining_time": "10h 1m 41s"} +{"eps": 0, "objective/kl": 18.71600723, "objective/entropy": -94.70143127, "objective/non_score_reward": -0.93580037, "objective/rlhf_reward": 9.9274807, "objective/scores": 10.875, "policy/approxkl_avg": 0.00083298, "policy/clipfrac_avg": 0.01079336, "loss/policy_avg": -0.29814595, "loss/value_avg": 0.8969987, "val/clipfrac_avg": 0.11709762, "policy/entropy_avg": 0.14975071, "val/ratio": 0.99026966, "val/ratio_var": 4.784e-05, "val/num_eos_tokens": 115, "lr": 8.12e-06, "episode": 6384, "epoch": 0.32242424, "global_step/max_steps": "399/1238", "percentage": "32.23%", "elapsed_time": "4h 45m 40s", "remaining_time": "10h 0m 43s"} +{"eps": 0, "objective/kl": 21.73818588, "objective/entropy": -178.26353455, "objective/non_score_reward": -1.08690941, "objective/rlhf_reward": 10.70996571, "objective/scores": 11.8125, "policy/approxkl_avg": 0.0007763, "policy/clipfrac_avg": 0.00494683, "loss/policy_avg": -1.01192045, "loss/value_avg": 1.05388033, "val/clipfrac_avg": 0.12041096, "policy/entropy_avg": 0.14596653, "val/ratio": 0.99141359, "val/ratio_var": 3.936e-05, "val/num_eos_tokens": 105, "lr": 8.11e-06, "episode": 6400, "epoch": 0.32323232, "global_step/max_steps": "400/1238", "percentage": "32.31%", "elapsed_time": "4h 47m 6s", "remaining_time": "10h 1m 29s"} +{"eps": 0, "objective/kl": 22.77383423, "objective/entropy": -49.77229309, "objective/non_score_reward": -1.13869178, "objective/rlhf_reward": 8.75193405, "objective/scores": 9.875, "policy/approxkl_avg": 0.00098812, "policy/clipfrac_avg": 0.00677518, "loss/policy_avg": 0.00763739, "loss/value_avg": 0.69518709, "val/clipfrac_avg": 0.19054016, "policy/entropy_avg": 0.18160057, "val/ratio": 0.98806083, "val/ratio_var": 3.766e-05, "val/num_eos_tokens": 9, "lr": 8.1e-06, "episode": 6416, "epoch": 0.3240404, "global_step/max_steps": "401/1238", "percentage": "32.39%", "elapsed_time": "4h 48m 4s", "remaining_time": "10h 1m 17s"} +{"eps": 0, "objective/kl": 22.06227493, "objective/entropy": -94.79197693, "objective/non_score_reward": -1.10311365, "objective/rlhf_reward": 11.67813683, "objective/scores": 12.75, "policy/approxkl_avg": 0.00076129, "policy/clipfrac_avg": 0.00852666, "loss/policy_avg": -0.21244872, "loss/value_avg": 0.52437854, "val/clipfrac_avg": 0.25014567, "policy/entropy_avg": 0.1675663, "val/ratio": 0.99090362, "val/ratio_var": 3.403e-05, "val/num_eos_tokens": 163, "lr": 8.09e-06, "episode": 6432, "epoch": 0.32484848, "global_step/max_steps": "402/1238", "percentage": "32.47%", "elapsed_time": "4h 48m 54s", "remaining_time": "10h 0m 48s"} +{"eps": 0, "objective/kl": 18.59261513, "objective/entropy": -119.76268005, "objective/non_score_reward": -0.92963076, "objective/rlhf_reward": 9.37505722, "objective/scores": 10.3125, "policy/approxkl_avg": 0.00095, "policy/clipfrac_avg": 0.00850007, "loss/policy_avg": 0.0641107, "loss/value_avg": 0.70028782, "val/clipfrac_avg": 0.0623731, "policy/entropy_avg": 0.15731883, "val/ratio": 0.99205971, "val/ratio_var": 3.085e-05, "val/num_eos_tokens": 601, "lr": 8.08e-06, "episode": 6448, "epoch": 0.32565657, "global_step/max_steps": "403/1238", "percentage": "32.55%", "elapsed_time": "4h 49m 37s", "remaining_time": "10h 0m 6s"} +{"eps": 0, "objective/kl": 20.83065033, "objective/entropy": -34.34098434, "objective/non_score_reward": -1.04153252, "objective/rlhf_reward": 8.93112373, "objective/scores": 10.0, "policy/approxkl_avg": 0.00118349, "policy/clipfrac_avg": 0.01117489, "loss/policy_avg": -0.07570268, "loss/value_avg": 0.50155425, "val/clipfrac_avg": 0.08182005, "policy/entropy_avg": 0.18660355, "val/ratio": 0.98766708, "val/ratio_var": 4.625e-05, "val/num_eos_tokens": 52, "lr": 8.06e-06, "episode": 6464, "epoch": 0.32646465, "global_step/max_steps": "404/1238", "percentage": "32.63%", "elapsed_time": "4h 50m 8s", "remaining_time": "9h 58m 56s"} +{"eps": 0, "objective/kl": 23.33823204, "objective/entropy": -124.38381195, "objective/non_score_reward": -1.1669116, "objective/rlhf_reward": 8.56746292, "objective/scores": 9.75, "policy/approxkl_avg": 0.00089511, "policy/clipfrac_avg": 0.0179115, "loss/policy_avg": 0.27639401, "loss/value_avg": 0.72849065, "val/clipfrac_avg": 0.06235784, "policy/entropy_avg": 0.14433575, "val/ratio": 0.99090469, "val/ratio_var": 2.75e-05, "val/num_eos_tokens": 413, "lr": 8.05e-06, "episode": 6480, "epoch": 0.32727273, "global_step/max_steps": "405/1238", "percentage": "32.71%", "elapsed_time": "4h 50m 47s", "remaining_time": "9h 58m 6s"} +{"eps": 0, "objective/kl": 23.43341064, "objective/entropy": -124.50070953, "objective/non_score_reward": -1.17167044, "objective/rlhf_reward": 8.89473534, "objective/scores": 10.0625, "policy/approxkl_avg": 0.00084795, "policy/clipfrac_avg": 0.01316243, "loss/policy_avg": -0.36677244, "loss/value_avg": 0.46131957, "val/clipfrac_avg": 0.14444213, "policy/entropy_avg": 0.19167137, "val/ratio": 0.9900524, "val/ratio_var": 6.207e-05, "val/num_eos_tokens": 311, "lr": 8.04e-06, "episode": 6496, "epoch": 0.32808081, "global_step/max_steps": "406/1238", "percentage": "32.79%", "elapsed_time": "4h 51m 23s", "remaining_time": "9h 57m 8s"} +{"eps": 0, "objective/kl": 19.72597504, "objective/entropy": -91.82843018, "objective/non_score_reward": -0.98629886, "objective/rlhf_reward": 10.45901394, "objective/scores": 11.4375, "policy/approxkl_avg": 0.00082226, "policy/clipfrac_avg": 0.01588718, "loss/policy_avg": 0.04725009, "loss/value_avg": 1.21890759, "val/clipfrac_avg": 0.16988257, "policy/entropy_avg": 0.13180411, "val/ratio": 0.99135005, "val/ratio_var": 4.945e-05, "val/num_eos_tokens": 286, "lr": 8.03e-06, "episode": 6512, "epoch": 0.32888889, "global_step/max_steps": "407/1238", "percentage": "32.88%", "elapsed_time": "4h 51m 54s", "remaining_time": "9h 56m 1s"} +{"eps": 0, "objective/kl": 22.47317505, "objective/entropy": -143.99177551, "objective/non_score_reward": -1.12365878, "objective/rlhf_reward": 10.82165337, "objective/scores": 11.9375, "policy/approxkl_avg": 0.00069033, "policy/clipfrac_avg": 0.02162604, "loss/policy_avg": -0.4653534, "loss/value_avg": 0.79269373, "val/clipfrac_avg": 0.14454639, "policy/entropy_avg": 0.16019592, "val/ratio": 0.99193132, "val/ratio_var": 4.091e-05, "val/num_eos_tokens": 448, "lr": 8.02e-06, "episode": 6528, "epoch": 0.32969697, "global_step/max_steps": "408/1238", "percentage": "32.96%", "elapsed_time": "4h 52m 52s", "remaining_time": "9h 55m 48s"} +{"eps": 0, "objective/kl": 22.80106354, "objective/entropy": -75.08297729, "objective/non_score_reward": -1.14005327, "objective/rlhf_reward": 9.3755722, "objective/scores": 10.5, "policy/approxkl_avg": 0.00122384, "policy/clipfrac_avg": 0.01396718, "loss/policy_avg": 0.07690079, "loss/value_avg": 0.2945565, "val/clipfrac_avg": 0.19421738, "policy/entropy_avg": 0.15617752, "val/ratio": 0.98936903, "val/ratio_var": 5.306e-05, "val/num_eos_tokens": 56, "lr": 8.01e-06, "episode": 6544, "epoch": 0.33050505, "global_step/max_steps": "409/1238", "percentage": "33.04%", "elapsed_time": "4h 53m 25s", "remaining_time": "9h 54m 44s"} +{"eps": 0, "objective/kl": 23.71062851, "objective/entropy": -171.51019287, "objective/non_score_reward": -1.18553162, "objective/rlhf_reward": 10.86915588, "objective/scores": 12.0625, "policy/approxkl_avg": 0.00092954, "policy/clipfrac_avg": 0.03016268, "loss/policy_avg": 0.10671385, "loss/value_avg": 0.24573182, "val/clipfrac_avg": 0.0707895, "policy/entropy_avg": 0.17933369, "val/ratio": 0.99171817, "val/ratio_var": 3.603e-05, "val/num_eos_tokens": 479, "lr": 8e-06, "episode": 6560, "epoch": 0.33131313, "global_step/max_steps": "410/1238", "percentage": "33.12%", "elapsed_time": "4h 54m 19s", "remaining_time": "9h 54m 24s"} +{"eps": 0, "objective/kl": 18.75413132, "objective/entropy": -92.83210754, "objective/non_score_reward": -0.93770647, "objective/rlhf_reward": 11.25370026, "objective/scores": 12.1875, "policy/approxkl_avg": 0.00083785, "policy/clipfrac_avg": 0.00801713, "loss/policy_avg": -0.28077972, "loss/value_avg": 1.3987999, "val/clipfrac_avg": 0.18445538, "policy/entropy_avg": 0.15318012, "val/ratio": 0.98984051, "val/ratio_var": 7.418e-05, "val/num_eos_tokens": 23, "lr": 7.99e-06, "episode": 6576, "epoch": 0.33212121, "global_step/max_steps": "411/1238", "percentage": "33.20%", "elapsed_time": "4h 54m 47s", "remaining_time": "9h 53m 10s"} +{"eps": 0, "objective/kl": 25.96450806, "objective/entropy": -66.04624939, "objective/non_score_reward": -1.2982254, "objective/rlhf_reward": 9.7798996, "objective/scores": 11.0625, "policy/approxkl_avg": 0.00100152, "policy/clipfrac_avg": 0.00993548, "loss/policy_avg": -0.18175235, "loss/value_avg": 0.18787694, "val/clipfrac_avg": 0.0434254, "policy/entropy_avg": 0.17697501, "val/ratio": 0.98832458, "val/ratio_var": 5.868e-05, "val/num_eos_tokens": 47, "lr": 7.98e-06, "episode": 6592, "epoch": 0.33292929, "global_step/max_steps": "412/1238", "percentage": "33.28%", "elapsed_time": "4h 55m 19s", "remaining_time": "9h 52m 5s"} +{"eps": 0, "objective/kl": 25.51277924, "objective/entropy": -34.08283234, "objective/non_score_reward": -1.27563894, "objective/rlhf_reward": 9.02904892, "objective/scores": 10.3125, "policy/approxkl_avg": 0.00108012, "policy/clipfrac_avg": 0.00930698, "loss/policy_avg": 0.22827569, "loss/value_avg": 0.52280796, "val/clipfrac_avg": 0.0479473, "policy/entropy_avg": 0.2053299, "val/ratio": 0.98741776, "val/ratio_var": 2.474e-05, "val/num_eos_tokens": 18, "lr": 7.97e-06, "episode": 6608, "epoch": 0.33373737, "global_step/max_steps": "413/1238", "percentage": "33.36%", "elapsed_time": "4h 56m 1s", "remaining_time": "9h 51m 20s"} +{"eps": 0, "objective/kl": 17.21203041, "objective/entropy": -88.73225403, "objective/non_score_reward": -0.86060154, "objective/rlhf_reward": 12.13158607, "objective/scores": 13.0, "policy/approxkl_avg": 0.00068784, "policy/clipfrac_avg": 0.00582766, "loss/policy_avg": -0.24807936, "loss/value_avg": 0.78812712, "val/clipfrac_avg": 0.22013108, "policy/entropy_avg": 0.12680221, "val/ratio": 0.99188685, "val/ratio_var": 6.46e-05, "val/num_eos_tokens": 113, "lr": 7.96e-06, "episode": 6624, "epoch": 0.33454545, "global_step/max_steps": "414/1238", "percentage": "33.44%", "elapsed_time": "4h 56m 33s", "remaining_time": "9h 50m 15s"} +{"eps": 0, "objective/kl": 22.45389557, "objective/entropy": -65.54212952, "objective/non_score_reward": -1.12269485, "objective/rlhf_reward": 11.28355503, "objective/scores": 12.375, "policy/approxkl_avg": 0.00085676, "policy/clipfrac_avg": 0.00504047, "loss/policy_avg": -0.00734615, "loss/value_avg": 1.34987056, "val/clipfrac_avg": 0.10788098, "policy/entropy_avg": 0.16246891, "val/ratio": 0.9893018, "val/ratio_var": 4.848e-05, "val/num_eos_tokens": 322, "lr": 7.95e-06, "episode": 6640, "epoch": 0.33535354, "global_step/max_steps": "415/1238", "percentage": "33.52%", "elapsed_time": "4h 57m 10s", "remaining_time": "9h 49m 20s"} +{"eps": 0, "objective/kl": 13.1745348, "objective/entropy": -25.82935715, "objective/non_score_reward": -0.65872681, "objective/rlhf_reward": 10.31392956, "objective/scores": 11.0, "policy/approxkl_avg": 0.00087465, "policy/clipfrac_avg": 0.00642186, "loss/policy_avg": 0.3142643, "loss/value_avg": 2.21596098, "val/clipfrac_avg": 0.15579936, "policy/entropy_avg": 0.19477272, "val/ratio": 0.99000823, "val/ratio_var": 9.775e-05, "val/num_eos_tokens": 7, "lr": 7.94e-06, "episode": 6656, "epoch": 0.33616162, "global_step/max_steps": "416/1238", "percentage": "33.60%", "elapsed_time": "4h 57m 36s", "remaining_time": "9h 48m 3s"} +{"eps": 0, "objective/kl": 20.85487747, "objective/entropy": -116.59674072, "objective/non_score_reward": -1.04274392, "objective/rlhf_reward": 13.40256882, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00062887, "policy/clipfrac_avg": 0.00324663, "loss/policy_avg": -0.57801598, "loss/value_avg": 0.74954563, "val/clipfrac_avg": 0.1060465, "policy/entropy_avg": 0.12189758, "val/ratio": 0.99188101, "val/ratio_var": 3.449e-05, "val/num_eos_tokens": 124, "lr": 7.93e-06, "episode": 6672, "epoch": 0.3369697, "global_step/max_steps": "417/1238", "percentage": "33.68%", "elapsed_time": "4h 58m 17s", "remaining_time": "9h 47m 17s"} +{"eps": 0, "objective/kl": 25.00245285, "objective/entropy": -129.16410828, "objective/non_score_reward": -1.25012279, "objective/rlhf_reward": 9.64831543, "objective/scores": 10.875, "policy/approxkl_avg": 0.00086851, "policy/clipfrac_avg": 0.00923302, "loss/policy_avg": 0.1208265, "loss/value_avg": 0.75480366, "val/clipfrac_avg": 0.14824736, "policy/entropy_avg": 0.16512156, "val/ratio": 0.98871624, "val/ratio_var": 7.21e-05, "val/num_eos_tokens": 431, "lr": 7.92e-06, "episode": 6688, "epoch": 0.33777778, "global_step/max_steps": "418/1238", "percentage": "33.76%", "elapsed_time": "4h 59m 17s", "remaining_time": "9h 47m 7s"} +{"eps": 0, "objective/kl": 17.61622047, "objective/entropy": -99.79671478, "objective/non_score_reward": -0.8808111, "objective/rlhf_reward": 10.90825081, "objective/scores": 11.8125, "policy/approxkl_avg": 0.00068185, "policy/clipfrac_avg": 0.01047616, "loss/policy_avg": 0.3398968, "loss/value_avg": 1.41889095, "val/clipfrac_avg": 0.19594175, "policy/entropy_avg": 0.14774263, "val/ratio": 0.99217558, "val/ratio_var": 3.494e-05, "val/num_eos_tokens": 87, "lr": 7.9e-06, "episode": 6704, "epoch": 0.33858586, "global_step/max_steps": "419/1238", "percentage": "33.84%", "elapsed_time": "4h 59m 46s", "remaining_time": "9h 45m 57s"} +{"eps": 0, "objective/kl": 20.07395935, "objective/entropy": -143.52662659, "objective/non_score_reward": -1.00369787, "objective/rlhf_reward": 11.24630165, "objective/scores": 12.25, "policy/approxkl_avg": 0.00079995, "policy/clipfrac_avg": 0.00589603, "loss/policy_avg": -0.20346837, "loss/value_avg": 1.35212779, "val/clipfrac_avg": 0.10764417, "policy/entropy_avg": 0.14706612, "val/ratio": 0.99001741, "val/ratio_var": 5.076e-05, "val/num_eos_tokens": 184, "lr": 7.89e-06, "episode": 6720, "epoch": 0.33939394, "global_step/max_steps": "420/1238", "percentage": "33.93%", "elapsed_time": "5h 0m 27s", "remaining_time": "9h 45m 10s"} +{"eps": 0, "objective/kl": 24.23324203, "objective/entropy": -125.03632355, "objective/non_score_reward": -1.21166217, "objective/rlhf_reward": 9.20240021, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00083043, "policy/clipfrac_avg": 0.00486622, "loss/policy_avg": 0.36388654, "loss/value_avg": 0.49889261, "val/clipfrac_avg": 0.11759979, "policy/entropy_avg": 0.15968847, "val/ratio": 0.99000871, "val/ratio_var": 5.003e-05, "val/num_eos_tokens": 266, "lr": 7.88e-06, "episode": 6736, "epoch": 0.34020202, "global_step/max_steps": "421/1238", "percentage": "34.01%", "elapsed_time": "5h 1m 0s", "remaining_time": "9h 44m 9s"} +{"eps": 0, "objective/kl": 24.24749374, "objective/entropy": -25.79072952, "objective/non_score_reward": -1.21237481, "objective/rlhf_reward": 10.97512531, "objective/scores": 12.1875, "policy/approxkl_avg": 0.00101154, "policy/clipfrac_avg": 0.00670176, "loss/policy_avg": 0.1830233, "loss/value_avg": 0.67552012, "val/clipfrac_avg": 0.14078917, "policy/entropy_avg": 0.16680622, "val/ratio": 0.98813927, "val/ratio_var": 6.585e-05, "val/num_eos_tokens": 30, "lr": 7.87e-06, "episode": 6752, "epoch": 0.3410101, "global_step/max_steps": "422/1238", "percentage": "34.09%", "elapsed_time": "5h 1m 39s", "remaining_time": "9h 43m 17s"} +{"eps": 0, "objective/kl": 23.44736481, "objective/entropy": -99.92378235, "objective/non_score_reward": -1.17236829, "objective/rlhf_reward": 10.062006, "objective/scores": 11.25, "policy/approxkl_avg": 0.00085159, "policy/clipfrac_avg": 0.01178953, "loss/policy_avg": 0.1767496, "loss/value_avg": 0.8422823, "val/clipfrac_avg": 0.07063313, "policy/entropy_avg": 0.1661377, "val/ratio": 0.99006528, "val/ratio_var": 2.773e-05, "val/num_eos_tokens": 82, "lr": 7.86e-06, "episode": 6768, "epoch": 0.34181818, "global_step/max_steps": "423/1238", "percentage": "34.17%", "elapsed_time": "5h 2m 21s", "remaining_time": "9h 42m 33s"} +{"eps": 0, "objective/kl": 22.92315292, "objective/entropy": -87.34598541, "objective/non_score_reward": -1.14615774, "objective/rlhf_reward": 10.05696678, "objective/scores": 11.1875, "policy/approxkl_avg": 0.00100464, "policy/clipfrac_avg": 0.01210677, "loss/policy_avg": -0.09848126, "loss/value_avg": 0.28399372, "val/clipfrac_avg": 0.06793521, "policy/entropy_avg": 0.15830231, "val/ratio": 0.98907334, "val/ratio_var": 4.018e-05, "val/num_eos_tokens": 316, "lr": 7.85e-06, "episode": 6784, "epoch": 0.34262626, "global_step/max_steps": "424/1238", "percentage": "34.25%", "elapsed_time": "5h 2m 56s", "remaining_time": "9h 41m 34s"} +{"eps": 0, "objective/kl": 20.97962952, "objective/entropy": -24.36560059, "objective/non_score_reward": -1.04898143, "objective/rlhf_reward": 10.31820679, "objective/scores": 11.375, "policy/approxkl_avg": 0.0009984, "policy/clipfrac_avg": 0.00608045, "loss/policy_avg": -0.43376964, "loss/value_avg": 1.939749, "val/clipfrac_avg": 0.10390697, "policy/entropy_avg": 0.17426896, "val/ratio": 0.99019396, "val/ratio_var": 5.642e-05, "val/num_eos_tokens": 217, "lr": 7.84e-06, "episode": 6800, "epoch": 0.34343434, "global_step/max_steps": "425/1238", "percentage": "34.33%", "elapsed_time": "5h 3m 20s", "remaining_time": "9h 40m 15s"} +{"eps": 0, "objective/kl": 27.88033867, "objective/entropy": -38.45948792, "objective/non_score_reward": -1.39401686, "objective/rlhf_reward": 9.11379623, "objective/scores": 10.5, "policy/approxkl_avg": 0.00097822, "policy/clipfrac_avg": 0.00647475, "loss/policy_avg": 0.00104612, "loss/value_avg": 0.22941998, "val/clipfrac_avg": 0.03179372, "policy/entropy_avg": 0.18066978, "val/ratio": 0.98816776, "val/ratio_var": 3.83e-05, "val/num_eos_tokens": 113, "lr": 7.83e-06, "episode": 6816, "epoch": 0.34424242, "global_step/max_steps": "426/1238", "percentage": "34.41%", "elapsed_time": "5h 3m 55s", "remaining_time": "9h 39m 18s"} +{"eps": 0, "objective/kl": 30.0983963, "objective/entropy": -61.58184052, "objective/non_score_reward": -1.50491977, "objective/rlhf_reward": 9.81539249, "objective/scores": 11.3125, "policy/approxkl_avg": 0.00108635, "policy/clipfrac_avg": 0.01059108, "loss/policy_avg": -0.07846694, "loss/value_avg": 0.26400119, "val/clipfrac_avg": 0.03756114, "policy/entropy_avg": 0.15493393, "val/ratio": 0.98974311, "val/ratio_var": 4.053e-05, "val/num_eos_tokens": 138, "lr": 7.82e-06, "episode": 6832, "epoch": 0.34505051, "global_step/max_steps": "427/1238", "percentage": "34.49%", "elapsed_time": "5h 4m 38s", "remaining_time": "9h 38m 36s"} +{"eps": 0, "objective/kl": 21.42774963, "objective/entropy": -48.85191345, "objective/non_score_reward": -1.07138741, "objective/rlhf_reward": 9.93642616, "objective/scores": 11.0, "policy/approxkl_avg": 0.00235914, "policy/clipfrac_avg": 0.01906696, "loss/policy_avg": -0.38922715, "loss/value_avg": 0.63401133, "val/clipfrac_avg": 0.08839963, "policy/entropy_avg": 0.20233774, "val/ratio": 0.9836812, "val/ratio_var": 0.00025593, "val/num_eos_tokens": 56, "lr": 7.81e-06, "episode": 6848, "epoch": 0.34585859, "global_step/max_steps": "428/1238", "percentage": "34.57%", "elapsed_time": "5h 5m 14s", "remaining_time": "9h 37m 39s"} +{"eps": 0, "objective/kl": 19.13961029, "objective/entropy": -94.49333191, "objective/non_score_reward": -0.95698047, "objective/rlhf_reward": 13.48051929, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00101495, "policy/clipfrac_avg": 0.00983855, "loss/policy_avg": -0.41618934, "loss/value_avg": 1.77342105, "val/clipfrac_avg": 0.18258625, "policy/entropy_avg": 0.16881979, "val/ratio": 0.98926222, "val/ratio_var": 8.774e-05, "val/num_eos_tokens": 84, "lr": 7.79e-06, "episode": 6864, "epoch": 0.34666667, "global_step/max_steps": "429/1238", "percentage": "34.65%", "elapsed_time": "5h 5m 54s", "remaining_time": "9h 36m 51s"} +{"eps": 0, "objective/kl": 28.28658676, "objective/entropy": -144.04450989, "objective/non_score_reward": -1.41432941, "objective/rlhf_reward": 11.10129547, "objective/scores": 12.5, "policy/approxkl_avg": 0.00077554, "policy/clipfrac_avg": 0.00578592, "loss/policy_avg": -0.36678922, "loss/value_avg": 0.39084461, "val/clipfrac_avg": 0.11318591, "policy/entropy_avg": 0.15067649, "val/ratio": 0.99023336, "val/ratio_var": 4.714e-05, "val/num_eos_tokens": 683, "lr": 7.78e-06, "episode": 6880, "epoch": 0.34747475, "global_step/max_steps": "430/1238", "percentage": "34.73%", "elapsed_time": "5h 6m 35s", "remaining_time": "9h 36m 6s"} +{"eps": 0, "objective/kl": 24.70610809, "objective/entropy": -88.11723328, "objective/non_score_reward": -1.23530531, "objective/rlhf_reward": 11.60844421, "objective/scores": 12.875, "policy/approxkl_avg": 0.00125058, "policy/clipfrac_avg": 0.01738021, "loss/policy_avg": -0.07360353, "loss/value_avg": 0.43915641, "val/clipfrac_avg": 0.088138, "policy/entropy_avg": 0.16494894, "val/ratio": 0.98865485, "val/ratio_var": 8.897e-05, "val/num_eos_tokens": 396, "lr": 7.77e-06, "episode": 6896, "epoch": 0.34828283, "global_step/max_steps": "431/1238", "percentage": "34.81%", "elapsed_time": "5h 7m 3s", "remaining_time": "9h 34m 56s"} +{"eps": 0, "objective/kl": 25.28207016, "objective/entropy": -98.53260803, "objective/non_score_reward": -1.26410341, "objective/rlhf_reward": 10.77495956, "objective/scores": 12.0625, "policy/approxkl_avg": 0.00094655, "policy/clipfrac_avg": 0.01756618, "loss/policy_avg": -0.39273804, "loss/value_avg": 0.74071181, "val/clipfrac_avg": 0.05415057, "policy/entropy_avg": 0.17182016, "val/ratio": 0.98874998, "val/ratio_var": 5.97e-05, "val/num_eos_tokens": 205, "lr": 7.76e-06, "episode": 6912, "epoch": 0.34909091, "global_step/max_steps": "432/1238", "percentage": "34.89%", "elapsed_time": "5h 7m 46s", "remaining_time": "9h 34m 14s"} +{"eps": 0, "objective/kl": 29.98111916, "objective/entropy": -105.76652527, "objective/non_score_reward": -1.49905598, "objective/rlhf_reward": 11.25094414, "objective/scores": 12.75, "policy/approxkl_avg": 0.00081968, "policy/clipfrac_avg": 0.01405583, "loss/policy_avg": 0.07162999, "loss/value_avg": 0.6784457, "val/clipfrac_avg": 0.10481824, "policy/entropy_avg": 0.16859722, "val/ratio": 0.98962498, "val/ratio_var": 6.031e-05, "val/num_eos_tokens": 508, "lr": 7.75e-06, "episode": 6928, "epoch": 0.34989899, "global_step/max_steps": "433/1238", "percentage": "34.98%", "elapsed_time": "5h 8m 20s", "remaining_time": "9h 33m 15s"} +{"eps": 0, "objective/kl": 24.6820755, "objective/entropy": -118.70050049, "objective/non_score_reward": -1.23410368, "objective/rlhf_reward": 10.75808334, "objective/scores": 12.0, "policy/approxkl_avg": 0.00077107, "policy/clipfrac_avg": 0.01288773, "loss/policy_avg": 0.2174176, "loss/value_avg": 0.48652163, "val/clipfrac_avg": 0.19917586, "policy/entropy_avg": 0.12623024, "val/ratio": 0.99221605, "val/ratio_var": 3.544e-05, "val/num_eos_tokens": 560, "lr": 7.74e-06, "episode": 6944, "epoch": 0.35070707, "global_step/max_steps": "434/1238", "percentage": "35.06%", "elapsed_time": "5h 9m 10s", "remaining_time": "9h 32m 46s"} +{"eps": 0, "objective/kl": 31.92902374, "objective/entropy": -127.72758484, "objective/non_score_reward": -1.59645116, "objective/rlhf_reward": 10.10667324, "objective/scores": 11.6875, "policy/approxkl_avg": 0.00089129, "policy/clipfrac_avg": 0.00693095, "loss/policy_avg": -0.07316741, "loss/value_avg": 0.22129662, "val/clipfrac_avg": 0.07383783, "policy/entropy_avg": 0.14632463, "val/ratio": 0.98961103, "val/ratio_var": 6.377e-05, "val/num_eos_tokens": 833, "lr": 7.73e-06, "episode": 6960, "epoch": 0.35151515, "global_step/max_steps": "435/1238", "percentage": "35.14%", "elapsed_time": "5h 9m 59s", "remaining_time": "9h 32m 14s"} +{"eps": 0, "objective/kl": 32.47252274, "objective/entropy": -99.71429443, "objective/non_score_reward": -1.62362599, "objective/rlhf_reward": 10.23574924, "objective/scores": 11.875, "policy/approxkl_avg": 0.00095191, "policy/clipfrac_avg": 0.01538163, "loss/policy_avg": 0.27702922, "loss/value_avg": 0.90741956, "val/clipfrac_avg": 0.30323496, "policy/entropy_avg": 0.17629117, "val/ratio": 0.98852527, "val/ratio_var": 7.608e-05, "val/num_eos_tokens": 165, "lr": 7.72e-06, "episode": 6976, "epoch": 0.35232323, "global_step/max_steps": "436/1238", "percentage": "35.22%", "elapsed_time": "5h 10m 41s", "remaining_time": "9h 31m 30s"} +{"eps": 0, "objective/kl": 29.33208656, "objective/entropy": -67.62409973, "objective/non_score_reward": -1.46660447, "objective/rlhf_reward": 12.58808327, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0009833, "policy/clipfrac_avg": 0.00630824, "loss/policy_avg": -0.10949158, "loss/value_avg": 0.7171697, "val/clipfrac_avg": 0.22335689, "policy/entropy_avg": 0.14640045, "val/ratio": 0.98930216, "val/ratio_var": 3.677e-05, "val/num_eos_tokens": 42, "lr": 7.71e-06, "episode": 6992, "epoch": 0.35313131, "global_step/max_steps": "437/1238", "percentage": "35.30%", "elapsed_time": "5h 11m 38s", "remaining_time": "9h 31m 14s"} +{"eps": 0, "objective/kl": 29.60206985, "objective/entropy": -34.5572052, "objective/non_score_reward": -1.48010349, "objective/rlhf_reward": 10.37927151, "objective/scores": 11.875, "policy/approxkl_avg": 0.00100615, "policy/clipfrac_avg": 0.00565114, "loss/policy_avg": 0.20945397, "loss/value_avg": 0.25410098, "val/clipfrac_avg": 0.14266336, "policy/entropy_avg": 0.17934394, "val/ratio": 0.98869675, "val/ratio_var": 4.758e-05, "val/num_eos_tokens": 142, "lr": 7.69e-06, "episode": 7008, "epoch": 0.35393939, "global_step/max_steps": "438/1238", "percentage": "35.38%", "elapsed_time": "5h 12m 19s", "remaining_time": "9h 30m 27s"} +{"eps": 0, "objective/kl": 25.04907608, "objective/entropy": -95.6053009, "objective/non_score_reward": -1.25245392, "objective/rlhf_reward": 10.74754524, "objective/scores": 12.0, "policy/approxkl_avg": 0.00301044, "policy/clipfrac_avg": 0.00949977, "loss/policy_avg": 0.04139136, "loss/value_avg": 0.80191201, "val/clipfrac_avg": 0.11603914, "policy/entropy_avg": 0.14387786, "val/ratio": 0.99052972, "val/ratio_var": 5.148e-05, "val/num_eos_tokens": 222, "lr": 7.68e-06, "episode": 7024, "epoch": 0.35474747, "global_step/max_steps": "439/1238", "percentage": "35.46%", "elapsed_time": "5h 12m 52s", "remaining_time": "9h 29m 26s"} +{"eps": 0, "objective/kl": 28.19591141, "objective/entropy": -84.95793152, "objective/non_score_reward": -1.40979564, "objective/rlhf_reward": 10.61364174, "objective/scores": 12.0, "policy/approxkl_avg": 0.00074768, "policy/clipfrac_avg": 0.00646547, "loss/policy_avg": 0.25317064, "loss/value_avg": 0.65145808, "val/clipfrac_avg": 0.16219607, "policy/entropy_avg": 0.12859344, "val/ratio": 0.99095976, "val/ratio_var": 5.391e-05, "val/num_eos_tokens": 104, "lr": 7.67e-06, "episode": 7040, "epoch": 0.35555556, "global_step/max_steps": "440/1238", "percentage": "35.54%", "elapsed_time": "5h 13m 27s", "remaining_time": "9h 28m 29s"} +{"eps": 0, "objective/kl": 38.44916916, "objective/entropy": -88.81119537, "objective/non_score_reward": -1.92245865, "objective/rlhf_reward": 10.08535385, "objective/scores": 12.0, "policy/approxkl_avg": 0.00097177, "policy/clipfrac_avg": 0.00741744, "loss/policy_avg": -0.04607268, "loss/value_avg": 0.24199238, "val/clipfrac_avg": 0.10574979, "policy/entropy_avg": 0.18393898, "val/ratio": 0.98761415, "val/ratio_var": 4.918e-05, "val/num_eos_tokens": 173, "lr": 7.66e-06, "episode": 7056, "epoch": 0.35636364, "global_step/max_steps": "441/1238", "percentage": "35.62%", "elapsed_time": "5h 14m 4s", "remaining_time": "9h 27m 36s"} +{"eps": 0, "objective/kl": 29.28141022, "objective/entropy": -92.62014771, "objective/non_score_reward": -1.46407056, "objective/rlhf_reward": 11.55155468, "objective/scores": 13.0, "policy/approxkl_avg": 0.00114339, "policy/clipfrac_avg": 0.01235822, "loss/policy_avg": -0.1001482, "loss/value_avg": 0.70477885, "val/clipfrac_avg": 0.07325706, "policy/entropy_avg": 0.16633868, "val/ratio": 0.98982388, "val/ratio_var": 5.311e-05, "val/num_eos_tokens": 292, "lr": 7.65e-06, "episode": 7072, "epoch": 0.35717172, "global_step/max_steps": "442/1238", "percentage": "35.70%", "elapsed_time": "5h 14m 47s", "remaining_time": "9h 26m 54s"} +{"eps": 0, "objective/kl": 30.10755539, "objective/entropy": -49.81909943, "objective/non_score_reward": -1.50537777, "objective/rlhf_reward": 9.26805878, "objective/scores": 10.75, "policy/approxkl_avg": 0.00125916, "policy/clipfrac_avg": 0.01565272, "loss/policy_avg": 0.26371089, "loss/value_avg": 0.88859206, "val/clipfrac_avg": 0.06298522, "policy/entropy_avg": 0.16948867, "val/ratio": 0.98828232, "val/ratio_var": 3.17e-05, "val/num_eos_tokens": 108, "lr": 7.64e-06, "episode": 7088, "epoch": 0.3579798, "global_step/max_steps": "443/1238", "percentage": "35.78%", "elapsed_time": "5h 15m 29s", "remaining_time": "9h 26m 10s"} +{"eps": 0, "objective/kl": 29.09910393, "objective/entropy": -128.95480347, "objective/non_score_reward": -1.45495522, "objective/rlhf_reward": 10.1700449, "objective/scores": 11.625, "policy/approxkl_avg": 0.00092313, "policy/clipfrac_avg": 0.00775393, "loss/policy_avg": 0.2935372, "loss/value_avg": 0.79652977, "val/clipfrac_avg": 0.12779459, "policy/entropy_avg": 0.17519093, "val/ratio": 0.98970687, "val/ratio_var": 4.803e-05, "val/num_eos_tokens": 712, "lr": 7.63e-06, "episode": 7104, "epoch": 0.35878788, "global_step/max_steps": "444/1238", "percentage": "35.86%", "elapsed_time": "5h 16m 20s", "remaining_time": "9h 25m 42s"} +{"eps": 0, "objective/kl": 43.96411514, "objective/entropy": -141.06298828, "objective/non_score_reward": -2.19820595, "objective/rlhf_reward": 8.77835655, "objective/scores": 11.0, "policy/approxkl_avg": 0.00105349, "policy/clipfrac_avg": 0.00662951, "loss/policy_avg": -0.10212292, "loss/value_avg": 0.41369742, "val/clipfrac_avg": 0.20534712, "policy/entropy_avg": 0.16738319, "val/ratio": 0.98934734, "val/ratio_var": 6.58e-05, "val/num_eos_tokens": 35, "lr": 7.61e-06, "episode": 7120, "epoch": 0.35959596, "global_step/max_steps": "445/1238", "percentage": "35.95%", "elapsed_time": "5h 18m 4s", "remaining_time": "9h 26m 48s"} +{"eps": 0, "objective/kl": 40.43344879, "objective/entropy": -137.25640869, "objective/non_score_reward": -2.02167273, "objective/rlhf_reward": 9.40801525, "objective/scores": 11.4375, "policy/approxkl_avg": 0.00111939, "policy/clipfrac_avg": 0.00953281, "loss/policy_avg": 0.04300054, "loss/value_avg": 0.28791785, "val/clipfrac_avg": 0.15842125, "policy/entropy_avg": 0.17898464, "val/ratio": 0.98815888, "val/ratio_var": 8.264e-05, "val/num_eos_tokens": 429, "lr": 7.6e-06, "episode": 7136, "epoch": 0.36040404, "global_step/max_steps": "446/1238", "percentage": "36.03%", "elapsed_time": "5h 19m 46s", "remaining_time": "9h 27m 50s"} +{"eps": 0, "objective/kl": 39.47451782, "objective/entropy": -107.86872864, "objective/non_score_reward": -1.97372603, "objective/rlhf_reward": 9.78408623, "objective/scores": 11.75, "policy/approxkl_avg": 0.00084151, "policy/clipfrac_avg": 0.00617805, "loss/policy_avg": -0.09768346, "loss/value_avg": 0.20441338, "val/clipfrac_avg": 0.1212514, "policy/entropy_avg": 0.15646553, "val/ratio": 0.99006343, "val/ratio_var": 4.89e-05, "val/num_eos_tokens": 35, "lr": 7.59e-06, "episode": 7152, "epoch": 0.36121212, "global_step/max_steps": "447/1238", "percentage": "36.11%", "elapsed_time": "5h 20m 18s", "remaining_time": "9h 26m 49s"} +{"eps": 0, "objective/kl": 34.3969574, "objective/entropy": -65.58972168, "objective/non_score_reward": -1.71984792, "objective/rlhf_reward": 10.53796482, "objective/scores": 12.25, "policy/approxkl_avg": 0.00099841, "policy/clipfrac_avg": 0.00585255, "loss/policy_avg": -0.23434095, "loss/value_avg": 0.98788077, "val/clipfrac_avg": 0.03700434, "policy/entropy_avg": 0.17050552, "val/ratio": 0.98790324, "val/ratio_var": 6.29e-05, "val/num_eos_tokens": 266, "lr": 7.58e-06, "episode": 7168, "epoch": 0.3620202, "global_step/max_steps": "448/1238", "percentage": "36.19%", "elapsed_time": "5h 20m 53s", "remaining_time": "9h 25m 51s"} +{"eps": 0, "objective/kl": 30.4707489, "objective/entropy": -42.06928635, "objective/non_score_reward": -1.5235374, "objective/rlhf_reward": 10.89833736, "objective/scores": 12.4375, "policy/approxkl_avg": 0.00085599, "policy/clipfrac_avg": 0.00813607, "loss/policy_avg": -0.09545287, "loss/value_avg": 0.35513487, "val/clipfrac_avg": 0.13745607, "policy/entropy_avg": 0.14156532, "val/ratio": 0.99171776, "val/ratio_var": 5.956e-05, "val/num_eos_tokens": 263, "lr": 7.57e-06, "episode": 7184, "epoch": 0.36282828, "global_step/max_steps": "449/1238", "percentage": "36.27%", "elapsed_time": "5h 21m 17s", "remaining_time": "9h 24m 35s"} +{"eps": 0, "objective/kl": 36.18313599, "objective/entropy": -166.65226746, "objective/non_score_reward": -1.80915678, "objective/rlhf_reward": 9.18303108, "objective/scores": 11.0, "policy/approxkl_avg": 0.00122102, "policy/clipfrac_avg": 0.01147709, "loss/policy_avg": -0.0989534, "loss/value_avg": 0.6456365, "val/clipfrac_avg": 0.14100708, "policy/entropy_avg": 0.15171528, "val/ratio": 0.98934305, "val/ratio_var": 6.88e-05, "val/num_eos_tokens": 196, "lr": 7.56e-06, "episode": 7200, "epoch": 0.36363636, "global_step/max_steps": "450/1238", "percentage": "36.35%", "elapsed_time": "5h 22m 57s", "remaining_time": "9h 25m 32s"} +{"eps": 0, "objective/kl": 28.16982269, "objective/entropy": -95.64318848, "objective/non_score_reward": -1.40849125, "objective/rlhf_reward": 10.37275887, "objective/scores": 11.75, "policy/approxkl_avg": 0.00093286, "policy/clipfrac_avg": 0.00823602, "loss/policy_avg": -0.03444557, "loss/value_avg": 1.06108272, "val/clipfrac_avg": 0.06177589, "policy/entropy_avg": 0.15565395, "val/ratio": 0.9893108, "val/ratio_var": 4.281e-05, "val/num_eos_tokens": 303, "lr": 7.55e-06, "episode": 7216, "epoch": 0.36444444, "global_step/max_steps": "451/1238", "percentage": "36.43%", "elapsed_time": "5h 23m 31s", "remaining_time": "9h 24m 33s"} +{"eps": 0, "objective/kl": 26.70556641, "objective/entropy": -65.11871338, "objective/non_score_reward": -1.33527827, "objective/rlhf_reward": 10.08659649, "objective/scores": 11.4375, "policy/approxkl_avg": 0.00102341, "policy/clipfrac_avg": 0.01305509, "loss/policy_avg": 0.2635814, "loss/value_avg": 0.52719271, "val/clipfrac_avg": 0.24446936, "policy/entropy_avg": 0.16281223, "val/ratio": 0.9888922, "val/ratio_var": 7.863e-05, "val/num_eos_tokens": 248, "lr": 7.53e-06, "episode": 7232, "epoch": 0.36525253, "global_step/max_steps": "452/1238", "percentage": "36.51%", "elapsed_time": "5h 23m 55s", "remaining_time": "9h 23m 17s"} +{"eps": 0, "objective/kl": 23.72340393, "objective/entropy": -100.4861908, "objective/non_score_reward": -1.18617022, "objective/rlhf_reward": 12.63414192, "objective/scores": 13.8125, "policy/approxkl_avg": 0.00103724, "policy/clipfrac_avg": 0.00965516, "loss/policy_avg": -0.20332152, "loss/value_avg": 2.09989405, "val/clipfrac_avg": 0.28355223, "policy/entropy_avg": 0.16978693, "val/ratio": 0.98854101, "val/ratio_var": 5.881e-05, "val/num_eos_tokens": 168, "lr": 7.52e-06, "episode": 7248, "epoch": 0.36606061, "global_step/max_steps": "453/1238", "percentage": "36.59%", "elapsed_time": "5h 24m 38s", "remaining_time": "9h 22m 34s"} +{"eps": 0, "objective/kl": 27.35384369, "objective/entropy": -121.3327713, "objective/non_score_reward": -1.36769223, "objective/rlhf_reward": 12.56980801, "objective/scores": 13.9375, "policy/approxkl_avg": 0.00090847, "policy/clipfrac_avg": 0.01056944, "loss/policy_avg": -0.2051644, "loss/value_avg": 0.57027447, "val/clipfrac_avg": 0.10168076, "policy/entropy_avg": 0.16514251, "val/ratio": 0.99044311, "val/ratio_var": 4.341e-05, "val/num_eos_tokens": 240, "lr": 7.51e-06, "episode": 7264, "epoch": 0.36686869, "global_step/max_steps": "454/1238", "percentage": "36.67%", "elapsed_time": "5h 25m 23s", "remaining_time": "9h 21m 54s"} +{"eps": 0, "objective/kl": 27.12935257, "objective/entropy": -94.46372223, "objective/non_score_reward": -1.3564676, "objective/rlhf_reward": 10.256814, "objective/scores": 11.625, "policy/approxkl_avg": 0.00101986, "policy/clipfrac_avg": 0.00805464, "loss/policy_avg": -0.11988246, "loss/value_avg": 0.24273254, "val/clipfrac_avg": 0.05403828, "policy/entropy_avg": 0.1758132, "val/ratio": 0.98846889, "val/ratio_var": 6.81e-05, "val/num_eos_tokens": 216, "lr": 7.5e-06, "episode": 7280, "epoch": 0.36767677, "global_step/max_steps": "455/1238", "percentage": "36.75%", "elapsed_time": "5h 26m 5s", "remaining_time": "9h 21m 10s"} +{"eps": 0, "objective/kl": 30.76184845, "objective/entropy": -71.2970047, "objective/non_score_reward": -1.53809249, "objective/rlhf_reward": 11.41503239, "objective/scores": 12.9375, "policy/approxkl_avg": 0.00106506, "policy/clipfrac_avg": 0.01344225, "loss/policy_avg": 0.15036239, "loss/value_avg": 1.35483289, "val/clipfrac_avg": 0.18505152, "policy/entropy_avg": 0.19586086, "val/ratio": 0.98620176, "val/ratio_var": 8.93e-05, "val/num_eos_tokens": 102, "lr": 7.49e-06, "episode": 7296, "epoch": 0.36848485, "global_step/max_steps": "456/1238", "percentage": "36.83%", "elapsed_time": "5h 26m 40s", "remaining_time": "9h 20m 13s"} +{"eps": 0, "objective/kl": 24.80374146, "objective/entropy": -110.9916687, "objective/non_score_reward": -1.24018717, "objective/rlhf_reward": 12.86137581, "objective/scores": 14.125, "policy/approxkl_avg": 0.0009098, "policy/clipfrac_avg": 0.01438397, "loss/policy_avg": -0.03644944, "loss/value_avg": 0.68133044, "val/clipfrac_avg": 0.14261322, "policy/entropy_avg": 0.14606905, "val/ratio": 0.99024922, "val/ratio_var": 5.003e-05, "val/num_eos_tokens": 396, "lr": 7.48e-06, "episode": 7312, "epoch": 0.36929293, "global_step/max_steps": "457/1238", "percentage": "36.91%", "elapsed_time": "5h 27m 20s", "remaining_time": "9h 19m 24s"} +{"eps": 0, "objective/kl": 22.95603943, "objective/entropy": -49.97304535, "objective/non_score_reward": -1.14780211, "objective/rlhf_reward": 10.87563515, "objective/scores": 12.0, "policy/approxkl_avg": 0.00101758, "policy/clipfrac_avg": 0.0112507, "loss/policy_avg": -0.08388766, "loss/value_avg": 0.990574, "val/clipfrac_avg": 0.21617578, "policy/entropy_avg": 0.20994115, "val/ratio": 0.98853564, "val/ratio_var": 7.954e-05, "val/num_eos_tokens": 59, "lr": 7.47e-06, "episode": 7328, "epoch": 0.37010101, "global_step/max_steps": "458/1238", "percentage": "37.00%", "elapsed_time": "5h 27m 46s", "remaining_time": "9h 18m 12s"} +{"eps": 0, "objective/kl": 27.69005013, "objective/entropy": -133.67175293, "objective/non_score_reward": -1.38450253, "objective/rlhf_reward": 10.36549759, "objective/scores": 11.75, "policy/approxkl_avg": 0.00083323, "policy/clipfrac_avg": 0.01680421, "loss/policy_avg": 0.12381671, "loss/value_avg": 0.41641736, "val/clipfrac_avg": 0.23695186, "policy/entropy_avg": 0.13964558, "val/ratio": 0.99241793, "val/ratio_var": 7.449e-05, "val/num_eos_tokens": 526, "lr": 7.45e-06, "episode": 7344, "epoch": 0.37090909, "global_step/max_steps": "459/1238", "percentage": "37.08%", "elapsed_time": "5h 28m 31s", "remaining_time": "9h 17m 33s"} +{"eps": 0, "objective/kl": 25.18273926, "objective/entropy": -37.12174988, "objective/non_score_reward": -1.25913692, "objective/rlhf_reward": 12.85023689, "objective/scores": 14.125, "policy/approxkl_avg": 0.00101756, "policy/clipfrac_avg": 0.01018021, "loss/policy_avg": -0.176779, "loss/value_avg": 2.02959657, "val/clipfrac_avg": 0.25308347, "policy/entropy_avg": 0.17505074, "val/ratio": 0.98835909, "val/ratio_var": 5.435e-05, "val/num_eos_tokens": 219, "lr": 7.44e-06, "episode": 7360, "epoch": 0.37171717, "global_step/max_steps": "460/1238", "percentage": "37.16%", "elapsed_time": "5h 29m 0s", "remaining_time": "9h 16m 27s"} +{"eps": 0, "objective/kl": 26.14715958, "objective/entropy": -70.19863892, "objective/non_score_reward": -1.30735791, "objective/rlhf_reward": 11.68482971, "objective/scores": 13.0, "policy/approxkl_avg": 0.00101347, "policy/clipfrac_avg": 0.01884063, "loss/policy_avg": 0.36454973, "loss/value_avg": 0.82008535, "val/clipfrac_avg": 0.15517488, "policy/entropy_avg": 0.178195, "val/ratio": 0.98932499, "val/ratio_var": 4.884e-05, "val/num_eos_tokens": 67, "lr": 7.43e-06, "episode": 7376, "epoch": 0.37252525, "global_step/max_steps": "461/1238", "percentage": "37.24%", "elapsed_time": "5h 29m 31s", "remaining_time": "9h 15m 24s"} +{"eps": 0, "objective/kl": 23.20512009, "objective/entropy": -105.91459656, "objective/non_score_reward": -1.16025603, "objective/rlhf_reward": 11.04286957, "objective/scores": 12.1875, "policy/approxkl_avg": 0.00088786, "policy/clipfrac_avg": 0.0075564, "loss/policy_avg": 0.31213555, "loss/value_avg": 0.54922521, "val/clipfrac_avg": 0.09090471, "policy/entropy_avg": 0.16464996, "val/ratio": 0.98896098, "val/ratio_var": 4.353e-05, "val/num_eos_tokens": 19, "lr": 7.42e-06, "episode": 7392, "epoch": 0.37333333, "global_step/max_steps": "462/1238", "percentage": "37.32%", "elapsed_time": "5h 30m 6s", "remaining_time": "9h 14m 27s"} +{"eps": 0, "objective/kl": 24.10005569, "objective/entropy": -67.86242676, "objective/non_score_reward": -1.2050029, "objective/rlhf_reward": 12.65437126, "objective/scores": 13.875, "policy/approxkl_avg": 0.00092614, "policy/clipfrac_avg": 0.01092603, "loss/policy_avg": -0.22459957, "loss/value_avg": 0.68713474, "val/clipfrac_avg": 0.12751663, "policy/entropy_avg": 0.17773855, "val/ratio": 0.98843682, "val/ratio_var": 7.627e-05, "val/num_eos_tokens": 290, "lr": 7.41e-06, "episode": 7408, "epoch": 0.37414141, "global_step/max_steps": "463/1238", "percentage": "37.40%", "elapsed_time": "5h 30m 38s", "remaining_time": "9h 13m 27s"} +{"eps": 0, "objective/kl": 25.08335495, "objective/entropy": -128.55480957, "objective/non_score_reward": -1.2541678, "objective/rlhf_reward": 10.96458244, "objective/scores": 12.25, "policy/approxkl_avg": 0.00090506, "policy/clipfrac_avg": 0.01150967, "loss/policy_avg": 0.10064421, "loss/value_avg": 0.42092299, "val/clipfrac_avg": 0.08698743, "policy/entropy_avg": 0.15951586, "val/ratio": 0.98957545, "val/ratio_var": 0.00011247, "val/num_eos_tokens": 233, "lr": 7.4e-06, "episode": 7424, "epoch": 0.37494949, "global_step/max_steps": "464/1238", "percentage": "37.48%", "elapsed_time": "5h 31m 44s", "remaining_time": "9h 13m 22s"} +{"eps": 0, "objective/kl": 23.57308197, "objective/entropy": -212.09448242, "objective/non_score_reward": -1.17865419, "objective/rlhf_reward": 11.71587658, "objective/scores": 12.875, "policy/approxkl_avg": 0.00070675, "policy/clipfrac_avg": 0.01169044, "loss/policy_avg": 0.18652041, "loss/value_avg": 0.69583762, "val/clipfrac_avg": 0.27865732, "policy/entropy_avg": 0.16191936, "val/ratio": 0.99156308, "val/ratio_var": 4.854e-05, "val/num_eos_tokens": 2015, "lr": 7.38e-06, "episode": 7440, "epoch": 0.37575758, "global_step/max_steps": "465/1238", "percentage": "37.56%", "elapsed_time": "5h 33m 28s", "remaining_time": "9h 14m 21s"} +{"eps": 0, "objective/kl": 32.34741592, "objective/entropy": -52.99230194, "objective/non_score_reward": -1.61737072, "objective/rlhf_reward": 11.49981689, "objective/scores": 13.125, "policy/approxkl_avg": 0.00102889, "policy/clipfrac_avg": 0.00455425, "loss/policy_avg": -0.02631384, "loss/value_avg": 0.31033474, "val/clipfrac_avg": 0.0315973, "policy/entropy_avg": 0.22930717, "val/ratio": 0.9858706, "val/ratio_var": 7.238e-05, "val/num_eos_tokens": 52, "lr": 7.37e-06, "episode": 7456, "epoch": 0.37656566, "global_step/max_steps": "466/1238", "percentage": "37.64%", "elapsed_time": "5h 34m 18s", "remaining_time": "9h 13m 49s"} +{"eps": 0, "objective/kl": 24.18475533, "objective/entropy": -167.13896179, "objective/non_score_reward": -1.20923769, "objective/rlhf_reward": 11.7165432, "objective/scores": 12.9375, "policy/approxkl_avg": 0.00081871, "policy/clipfrac_avg": 0.01295806, "loss/policy_avg": -0.17372929, "loss/value_avg": 0.49365771, "val/clipfrac_avg": 0.12412195, "policy/entropy_avg": 0.21362352, "val/ratio": 0.98991579, "val/ratio_var": 5.197e-05, "val/num_eos_tokens": 789, "lr": 7.36e-06, "episode": 7472, "epoch": 0.37737374, "global_step/max_steps": "467/1238", "percentage": "37.72%", "elapsed_time": "5h 35m 35s", "remaining_time": "9h 14m 3s"} +{"eps": 0, "objective/kl": 26.60739899, "objective/entropy": -99.32038879, "objective/non_score_reward": -1.33036995, "objective/rlhf_reward": 10.39619255, "objective/scores": 11.75, "policy/approxkl_avg": 0.00092526, "policy/clipfrac_avg": 0.00594138, "loss/policy_avg": -0.2519969, "loss/value_avg": 0.64790452, "val/clipfrac_avg": 0.08323414, "policy/entropy_avg": 0.18393898, "val/ratio": 0.98807776, "val/ratio_var": 0.00010124, "val/num_eos_tokens": 578, "lr": 7.35e-06, "episode": 7488, "epoch": 0.37818182, "global_step/max_steps": "468/1238", "percentage": "37.80%", "elapsed_time": "5h 36m 31s", "remaining_time": "9h 13m 40s"} +{"eps": 0, "objective/kl": 28.44561958, "objective/entropy": -113.41480255, "objective/non_score_reward": -1.42228091, "objective/rlhf_reward": 10.58553123, "objective/scores": 12.0, "policy/approxkl_avg": 0.00106993, "policy/clipfrac_avg": 0.02114877, "loss/policy_avg": 0.11465664, "loss/value_avg": 0.2218267, "val/clipfrac_avg": 0.05555972, "policy/entropy_avg": 0.17717171, "val/ratio": 0.98800051, "val/ratio_var": 5.972e-05, "val/num_eos_tokens": 301, "lr": 7.34e-06, "episode": 7504, "epoch": 0.3789899, "global_step/max_steps": "469/1238", "percentage": "37.88%", "elapsed_time": "5h 37m 30s", "remaining_time": "9h 13m 23s"} +{"eps": 0, "objective/kl": 24.17656898, "objective/entropy": -66.94224548, "objective/non_score_reward": -1.20882845, "objective/rlhf_reward": 12.02554703, "objective/scores": 13.25, "policy/approxkl_avg": 0.00087491, "policy/clipfrac_avg": 0.00713921, "loss/policy_avg": -0.16887972, "loss/value_avg": 1.12603879, "val/clipfrac_avg": 0.13150881, "policy/entropy_avg": 0.18168145, "val/ratio": 0.98968929, "val/ratio_var": 5.284e-05, "val/num_eos_tokens": 267, "lr": 7.32e-06, "episode": 7520, "epoch": 0.37979798, "global_step/max_steps": "470/1238", "percentage": "37.96%", "elapsed_time": "5h 38m 11s", "remaining_time": "9h 12m 37s"} +{"eps": 0, "objective/kl": 17.82182312, "objective/entropy": -28.33433723, "objective/non_score_reward": -0.89109111, "objective/rlhf_reward": 10.92140865, "objective/scores": 11.8125, "policy/approxkl_avg": 0.00100494, "policy/clipfrac_avg": 0.00611565, "loss/policy_avg": -0.1819143, "loss/value_avg": 0.7196762, "val/clipfrac_avg": 0.18608367, "policy/entropy_avg": 0.1875267, "val/ratio": 0.98810965, "val/ratio_var": 4.829e-05, "val/num_eos_tokens": 99, "lr": 7.31e-06, "episode": 7536, "epoch": 0.38060606, "global_step/max_steps": "471/1238", "percentage": "38.05%", "elapsed_time": "5h 38m 43s", "remaining_time": "9h 11m 35s"} +{"eps": 0, "objective/kl": 29.08966446, "objective/entropy": -104.8346405, "objective/non_score_reward": -1.45448315, "objective/rlhf_reward": 10.11582947, "objective/scores": 11.5625, "policy/approxkl_avg": 0.00088252, "policy/clipfrac_avg": 0.01230643, "loss/policy_avg": -0.12739496, "loss/value_avg": 0.68064409, "val/clipfrac_avg": 0.15843809, "policy/entropy_avg": 0.16717923, "val/ratio": 0.98944038, "val/ratio_var": 6.355e-05, "val/num_eos_tokens": 160, "lr": 7.3e-06, "episode": 7552, "epoch": 0.38141414, "global_step/max_steps": "472/1238", "percentage": "38.13%", "elapsed_time": "5h 39m 23s", "remaining_time": "9h 10m 46s"} +{"eps": 0, "objective/kl": 27.92177963, "objective/entropy": -16.5300827, "objective/non_score_reward": -1.39608896, "objective/rlhf_reward": 11.99453545, "objective/scores": 13.375, "policy/approxkl_avg": 0.00120451, "policy/clipfrac_avg": 0.00949463, "loss/policy_avg": 0.01314192, "loss/value_avg": 1.92228746, "val/clipfrac_avg": 0.10058106, "policy/entropy_avg": 0.22336864, "val/ratio": 0.98587143, "val/ratio_var": 6.286e-05, "val/num_eos_tokens": 123, "lr": 7.29e-06, "episode": 7568, "epoch": 0.38222222, "global_step/max_steps": "473/1238", "percentage": "38.21%", "elapsed_time": "5h 40m 10s", "remaining_time": "9h 10m 11s"} +{"eps": 0, "objective/kl": 33.5030098, "objective/entropy": -98.81828308, "objective/non_score_reward": -1.67515051, "objective/rlhf_reward": 11.56703758, "objective/scores": 13.25, "policy/approxkl_avg": 0.00082979, "policy/clipfrac_avg": 0.0088504, "loss/policy_avg": -0.28272492, "loss/value_avg": 0.53602231, "val/clipfrac_avg": 0.19075023, "policy/entropy_avg": 0.15388584, "val/ratio": 0.9900673, "val/ratio_var": 5.372e-05, "val/num_eos_tokens": 412, "lr": 7.28e-06, "episode": 7584, "epoch": 0.3830303, "global_step/max_steps": "474/1238", "percentage": "38.29%", "elapsed_time": "5h 40m 48s", "remaining_time": "9h 9m 18s"} +{"eps": 0, "objective/kl": 32.43540573, "objective/entropy": -70.269104, "objective/non_score_reward": -1.62177038, "objective/rlhf_reward": 11.59697914, "objective/scores": 13.25, "policy/approxkl_avg": 0.00091853, "policy/clipfrac_avg": 0.01303721, "loss/policy_avg": -0.2495127, "loss/value_avg": 0.32451302, "val/clipfrac_avg": 0.07557435, "policy/entropy_avg": 0.17321825, "val/ratio": 0.99027586, "val/ratio_var": 6.953e-05, "val/num_eos_tokens": 391, "lr": 7.27e-06, "episode": 7600, "epoch": 0.38383838, "global_step/max_steps": "475/1238", "percentage": "38.37%", "elapsed_time": "5h 41m 25s", "remaining_time": "9h 8m 25s"} +{"eps": 0, "objective/kl": 33.66416931, "objective/entropy": -75.98827362, "objective/non_score_reward": -1.68320835, "objective/rlhf_reward": 10.51210403, "objective/scores": 12.1875, "policy/approxkl_avg": 0.000912, "policy/clipfrac_avg": 0.00784029, "loss/policy_avg": -9.876e-05, "loss/value_avg": 0.57701945, "val/clipfrac_avg": 0.17109884, "policy/entropy_avg": 0.14813995, "val/ratio": 0.98977065, "val/ratio_var": 8.284e-05, "val/num_eos_tokens": 441, "lr": 7.25e-06, "episode": 7616, "epoch": 0.38464646, "global_step/max_steps": "476/1238", "percentage": "38.45%", "elapsed_time": "5h 42m 4s", "remaining_time": "9h 7m 35s"} +{"eps": 0, "objective/kl": 30.79873657, "objective/entropy": -107.81949615, "objective/non_score_reward": -1.5399369, "objective/rlhf_reward": 10.46006393, "objective/scores": 12.0, "policy/approxkl_avg": 0.00089137, "policy/clipfrac_avg": 0.01428598, "loss/policy_avg": -0.05202428, "loss/value_avg": 0.76978016, "val/clipfrac_avg": 0.13049725, "policy/entropy_avg": 0.14296913, "val/ratio": 0.99150032, "val/ratio_var": 3.443e-05, "val/num_eos_tokens": 247, "lr": 7.24e-06, "episode": 7632, "epoch": 0.38545455, "global_step/max_steps": "477/1238", "percentage": "38.53%", "elapsed_time": "5h 42m 44s", "remaining_time": "9h 6m 48s"} +{"eps": 0, "objective/kl": 19.900383, "objective/entropy": -69.23106384, "objective/non_score_reward": -0.9950192, "objective/rlhf_reward": 13.98935604, "objective/scores": 15.0, "policy/approxkl_avg": 0.00086115, "policy/clipfrac_avg": 0.00837149, "loss/policy_avg": -0.12627719, "loss/value_avg": 0.48913229, "val/clipfrac_avg": 0.08708248, "policy/entropy_avg": 0.13306999, "val/ratio": 0.99163759, "val/ratio_var": 4.943e-05, "val/num_eos_tokens": 13, "lr": 7.23e-06, "episode": 7648, "epoch": 0.38626263, "global_step/max_steps": "478/1238", "percentage": "38.61%", "elapsed_time": "5h 43m 8s", "remaining_time": "9h 5m 34s"} +{"eps": 0, "objective/kl": 26.03585052, "objective/entropy": -116.11354065, "objective/non_score_reward": -1.3017925, "objective/rlhf_reward": 10.33102036, "objective/scores": 11.625, "policy/approxkl_avg": 0.00088724, "policy/clipfrac_avg": 0.0121951, "loss/policy_avg": 0.13281304, "loss/value_avg": 1.58233166, "val/clipfrac_avg": 0.13728482, "policy/entropy_avg": 0.1528163, "val/ratio": 0.98945963, "val/ratio_var": 9.663e-05, "val/num_eos_tokens": 93, "lr": 7.22e-06, "episode": 7664, "epoch": 0.38707071, "global_step/max_steps": "479/1238", "percentage": "38.69%", "elapsed_time": "5h 44m 0s", "remaining_time": "9h 5m 6s"} +{"eps": 0, "objective/kl": 28.44430161, "objective/entropy": -102.0567627, "objective/non_score_reward": -1.42221522, "objective/rlhf_reward": 10.90591049, "objective/scores": 12.3125, "policy/approxkl_avg": 0.0007429, "policy/clipfrac_avg": 0.01388947, "loss/policy_avg": 1.05332124, "loss/value_avg": 1.33237123, "val/clipfrac_avg": 0.10630576, "policy/entropy_avg": 0.16810226, "val/ratio": 0.99027443, "val/ratio_var": 4.415e-05, "val/num_eos_tokens": 276, "lr": 7.21e-06, "episode": 7680, "epoch": 0.38787879, "global_step/max_steps": "480/1238", "percentage": "38.77%", "elapsed_time": "5h 44m 35s", "remaining_time": "9h 4m 10s"} +{"eps": 0, "objective/kl": 34.4516716, "objective/entropy": -135.23007202, "objective/non_score_reward": -1.72258365, "objective/rlhf_reward": 10.22272873, "objective/scores": 11.9375, "policy/approxkl_avg": 0.00118507, "policy/clipfrac_avg": 0.01151323, "loss/policy_avg": -0.075775, "loss/value_avg": 0.18845658, "val/clipfrac_avg": 0.01749832, "policy/entropy_avg": 0.18829852, "val/ratio": 0.98701394, "val/ratio_var": 6.372e-05, "val/num_eos_tokens": 1018, "lr": 7.19e-06, "episode": 7696, "epoch": 0.38868687, "global_step/max_steps": "481/1238", "percentage": "38.85%", "elapsed_time": "5h 45m 29s", "remaining_time": "9h 3m 44s"} +{"eps": 0, "objective/kl": 30.6765976, "objective/entropy": -64.4817276, "objective/non_score_reward": -1.53382993, "objective/rlhf_reward": 10.42710781, "objective/scores": 11.9375, "policy/approxkl_avg": 0.00089493, "policy/clipfrac_avg": 0.01055396, "loss/policy_avg": 0.03054455, "loss/value_avg": 0.34176174, "val/clipfrac_avg": 0.05515262, "policy/entropy_avg": 0.1752224, "val/ratio": 0.98915672, "val/ratio_var": 4.994e-05, "val/num_eos_tokens": 151, "lr": 7.18e-06, "episode": 7712, "epoch": 0.38949495, "global_step/max_steps": "482/1238", "percentage": "38.93%", "elapsed_time": "5h 46m 4s", "remaining_time": "9h 2m 48s"} +{"eps": 0, "objective/kl": 37.32021713, "objective/entropy": -52.11445618, "objective/non_score_reward": -1.8660109, "objective/rlhf_reward": 10.14180183, "objective/scores": 12.0, "policy/approxkl_avg": 0.00091628, "policy/clipfrac_avg": 0.00679036, "loss/policy_avg": -0.09279122, "loss/value_avg": 0.98249751, "val/clipfrac_avg": 0.06959789, "policy/entropy_avg": 0.19576645, "val/ratio": 0.98836744, "val/ratio_var": 5.33e-05, "val/num_eos_tokens": 63, "lr": 7.17e-06, "episode": 7728, "epoch": 0.39030303, "global_step/max_steps": "483/1238", "percentage": "39.01%", "elapsed_time": "5h 46m 45s", "remaining_time": "9h 2m 2s"} +{"eps": 0, "objective/kl": 40.67700195, "objective/entropy": -79.94268799, "objective/non_score_reward": -2.03385019, "objective/rlhf_reward": 11.00521278, "objective/scores": 13.0625, "policy/approxkl_avg": 0.00093703, "policy/clipfrac_avg": 0.00797117, "loss/policy_avg": -0.17013842, "loss/value_avg": 0.34701002, "val/clipfrac_avg": 0.05036664, "policy/entropy_avg": 0.17804527, "val/ratio": 0.98820639, "val/ratio_var": 3.965e-05, "val/num_eos_tokens": 591, "lr": 7.16e-06, "episode": 7744, "epoch": 0.39111111, "global_step/max_steps": "484/1238", "percentage": "39.10%", "elapsed_time": "5h 47m 28s", "remaining_time": "9h 1m 18s"} +{"eps": 0, "objective/kl": 33.60002899, "objective/entropy": -111.34992218, "objective/non_score_reward": -1.6800015, "objective/rlhf_reward": 9.39031029, "objective/scores": 11.0625, "policy/approxkl_avg": 0.00078605, "policy/clipfrac_avg": 0.00748593, "loss/policy_avg": 0.08009683, "loss/value_avg": 0.88024747, "val/clipfrac_avg": 0.16233416, "policy/entropy_avg": 0.15513873, "val/ratio": 0.99123859, "val/ratio_var": 4.485e-05, "val/num_eos_tokens": 219, "lr": 7.15e-06, "episode": 7760, "epoch": 0.39191919, "global_step/max_steps": "485/1238", "percentage": "39.18%", "elapsed_time": "5h 48m 1s", "remaining_time": "9h 0m 20s"} +{"eps": 0, "objective/kl": 34.01870728, "objective/entropy": -130.91871643, "objective/non_score_reward": -1.70093536, "objective/rlhf_reward": 10.59203339, "objective/scores": 12.3125, "policy/approxkl_avg": 0.00078681, "policy/clipfrac_avg": 0.01162111, "loss/policy_avg": 0.31785762, "loss/value_avg": 0.59200597, "val/clipfrac_avg": 0.08654517, "policy/entropy_avg": 0.1484971, "val/ratio": 0.99074912, "val/ratio_var": 4.032e-05, "val/num_eos_tokens": 387, "lr": 7.13e-06, "episode": 7776, "epoch": 0.39272727, "global_step/max_steps": "486/1238", "percentage": "39.26%", "elapsed_time": "5h 49m 3s", "remaining_time": "9h 0m 6s"} +{"eps": 0, "objective/kl": 30.16710091, "objective/entropy": -101.36183167, "objective/non_score_reward": -1.50835514, "objective/rlhf_reward": 10.51508236, "objective/scores": 12.0, "policy/approxkl_avg": 0.00111754, "policy/clipfrac_avg": 0.02089299, "loss/policy_avg": 0.56711853, "loss/value_avg": 0.62176645, "val/clipfrac_avg": 0.11080241, "policy/entropy_avg": 0.17551613, "val/ratio": 0.98857242, "val/ratio_var": 5.203e-05, "val/num_eos_tokens": 686, "lr": 7.12e-06, "episode": 7792, "epoch": 0.39353535, "global_step/max_steps": "487/1238", "percentage": "39.34%", "elapsed_time": "5h 49m 41s", "remaining_time": "8h 59m 14s"} +{"eps": 0, "objective/kl": 34.61153793, "objective/entropy": -156.5941925, "objective/non_score_reward": -1.73057687, "objective/rlhf_reward": 10.26161003, "objective/scores": 12.0, "policy/approxkl_avg": 0.0011252, "policy/clipfrac_avg": 0.00650518, "loss/policy_avg": -0.12260817, "loss/value_avg": 0.37460604, "val/clipfrac_avg": 0.07414089, "policy/entropy_avg": 0.21551895, "val/ratio": 0.98726618, "val/ratio_var": 7.019e-05, "val/num_eos_tokens": 29, "lr": 7.11e-06, "episode": 7808, "epoch": 0.39434343, "global_step/max_steps": "488/1238", "percentage": "39.42%", "elapsed_time": "5h 50m 35s", "remaining_time": "8h 58m 48s"} +{"eps": 0, "objective/kl": 35.93262482, "objective/entropy": -90.52662659, "objective/non_score_reward": -1.7966311, "objective/rlhf_reward": 9.00805664, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00089329, "policy/clipfrac_avg": 0.00704073, "loss/policy_avg": -0.03806201, "loss/value_avg": 0.51515836, "val/clipfrac_avg": 0.09701869, "policy/entropy_avg": 0.15955353, "val/ratio": 0.99025321, "val/ratio_var": 5.96e-05, "val/num_eos_tokens": 121, "lr": 7.1e-06, "episode": 7824, "epoch": 0.39515152, "global_step/max_steps": "489/1238", "percentage": "39.50%", "elapsed_time": "5h 51m 23s", "remaining_time": "8h 58m 14s"} +{"eps": 0, "objective/kl": 35.31539917, "objective/entropy": -107.76602173, "objective/non_score_reward": -1.76576996, "objective/rlhf_reward": 9.81235504, "objective/scores": 11.5625, "policy/approxkl_avg": 0.00094236, "policy/clipfrac_avg": 0.01131567, "loss/policy_avg": 0.18260059, "loss/value_avg": 1.15949214, "val/clipfrac_avg": 0.2163913, "policy/entropy_avg": 0.15435028, "val/ratio": 0.98889601, "val/ratio_var": 6.885e-05, "val/num_eos_tokens": 486, "lr": 7.08e-06, "episode": 7840, "epoch": 0.3959596, "global_step/max_steps": "490/1238", "percentage": "39.58%", "elapsed_time": "5h 52m 18s", "remaining_time": "8h 57m 48s"} +{"eps": 0, "objective/kl": 33.98522186, "objective/entropy": -41.98553085, "objective/non_score_reward": -1.69926107, "objective/rlhf_reward": 12.58980179, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00100926, "policy/clipfrac_avg": 0.01904711, "loss/policy_avg": -0.09957186, "loss/value_avg": 0.72608733, "val/clipfrac_avg": 0.15300216, "policy/entropy_avg": 0.15885639, "val/ratio": 0.98957324, "val/ratio_var": 4.709e-05, "val/num_eos_tokens": 182, "lr": 7.07e-06, "episode": 7856, "epoch": 0.39676768, "global_step/max_steps": "491/1238", "percentage": "39.66%", "elapsed_time": "5h 52m 49s", "remaining_time": "8h 56m 46s"} +{"eps": 0, "objective/kl": 24.98674965, "objective/entropy": -34.53561401, "objective/non_score_reward": -1.24933755, "objective/rlhf_reward": 11.3600378, "objective/scores": 12.625, "policy/approxkl_avg": 0.00124012, "policy/clipfrac_avg": 0.01449822, "loss/policy_avg": -0.07856579, "loss/value_avg": 0.40496874, "val/clipfrac_avg": 0.07531505, "policy/entropy_avg": 0.18246603, "val/ratio": 0.98802722, "val/ratio_var": 9.475e-05, "val/num_eos_tokens": 198, "lr": 7.06e-06, "episode": 7872, "epoch": 0.39757576, "global_step/max_steps": "492/1238", "percentage": "39.74%", "elapsed_time": "5h 53m 14s", "remaining_time": "8h 55m 36s"} +{"eps": 0, "objective/kl": 37.40539551, "objective/entropy": -74.6033783, "objective/non_score_reward": -1.87026978, "objective/rlhf_reward": 11.59066772, "objective/scores": 13.4375, "policy/approxkl_avg": 0.00089025, "policy/clipfrac_avg": 0.00807363, "loss/policy_avg": 0.06024223, "loss/value_avg": 0.333332, "val/clipfrac_avg": 0.05526703, "policy/entropy_avg": 0.16165543, "val/ratio": 0.99032068, "val/ratio_var": 3.266e-05, "val/num_eos_tokens": 447, "lr": 7.05e-06, "episode": 7888, "epoch": 0.39838384, "global_step/max_steps": "493/1238", "percentage": "39.82%", "elapsed_time": "5h 53m 59s", "remaining_time": "8h 54m 56s"} +{"eps": 0, "objective/kl": 37.19384766, "objective/entropy": -54.44144821, "objective/non_score_reward": -1.85969234, "objective/rlhf_reward": 14.31999493, "objective/scores": 16.125, "policy/approxkl_avg": 0.00126639, "policy/clipfrac_avg": 0.0102781, "loss/policy_avg": -0.06525195, "loss/value_avg": 0.67688644, "val/clipfrac_avg": 0.33751017, "policy/entropy_avg": 0.19942546, "val/ratio": 0.98598564, "val/ratio_var": 8.737e-05, "val/num_eos_tokens": 5, "lr": 7.04e-06, "episode": 7904, "epoch": 0.39919192, "global_step/max_steps": "494/1238", "percentage": "39.90%", "elapsed_time": "5h 55m 7s", "remaining_time": "8h 54m 50s"} +{"eps": 0, "objective/kl": 34.13693619, "objective/entropy": -161.62776184, "objective/non_score_reward": -1.70684695, "objective/rlhf_reward": 11.64471626, "objective/scores": 13.375, "policy/approxkl_avg": 0.00144677, "policy/clipfrac_avg": 0.0114209, "loss/policy_avg": -0.1597873, "loss/value_avg": 0.42206752, "val/clipfrac_avg": 0.07222696, "policy/entropy_avg": 0.13841856, "val/ratio": 0.98960042, "val/ratio_var": 0.00014254, "val/num_eos_tokens": 325, "lr": 7.02e-06, "episode": 7920, "epoch": 0.4, "global_step/max_steps": "495/1238", "percentage": "39.98%", "elapsed_time": "5h 56m 46s", "remaining_time": "8h 55m 31s"} +{"eps": 0, "objective/kl": 33.55063629, "objective/entropy": -129.15763855, "objective/non_score_reward": -1.67753172, "objective/rlhf_reward": 10.93770218, "objective/scores": 12.625, "policy/approxkl_avg": 0.00114749, "policy/clipfrac_avg": 0.01719671, "loss/policy_avg": -0.69860673, "loss/value_avg": 1.37642336, "val/clipfrac_avg": 0.18510054, "policy/entropy_avg": 0.20726132, "val/ratio": 0.98874766, "val/ratio_var": 9.583e-05, "val/num_eos_tokens": 428, "lr": 7.01e-06, "episode": 7936, "epoch": 0.40080808, "global_step/max_steps": "496/1238", "percentage": "40.06%", "elapsed_time": "5h 57m 26s", "remaining_time": "8h 54m 43s"} +{"eps": 0, "objective/kl": 32.41982651, "objective/entropy": -98.1259613, "objective/non_score_reward": -1.62099147, "objective/rlhf_reward": 11.33994579, "objective/scores": 12.9375, "policy/approxkl_avg": 0.00085711, "policy/clipfrac_avg": 0.0082765, "loss/policy_avg": -0.18637818, "loss/value_avg": 0.78860956, "val/clipfrac_avg": 0.04437315, "policy/entropy_avg": 0.15638733, "val/ratio": 0.98978037, "val/ratio_var": 3.502e-05, "val/num_eos_tokens": 728, "lr": 7e-06, "episode": 7952, "epoch": 0.40161616, "global_step/max_steps": "497/1238", "percentage": "40.15%", "elapsed_time": "5h 58m 9s", "remaining_time": "8h 53m 59s"} +{"eps": 0, "objective/kl": 27.43330383, "objective/entropy": -148.50958252, "objective/non_score_reward": -1.37166524, "objective/rlhf_reward": 11.15177155, "objective/scores": 12.5, "policy/approxkl_avg": 0.00125145, "policy/clipfrac_avg": 0.02511934, "loss/policy_avg": 0.55545503, "loss/value_avg": 0.98460829, "val/clipfrac_avg": 0.09154287, "policy/entropy_avg": 0.08732831, "val/ratio": 0.99407452, "val/ratio_var": 3.361e-05, "val/num_eos_tokens": 47, "lr": 6.99e-06, "episode": 7968, "epoch": 0.40242424, "global_step/max_steps": "498/1238", "percentage": "40.23%", "elapsed_time": "5h 59m 13s", "remaining_time": "8h 53m 47s"} +{"eps": 0, "objective/kl": 34.42153931, "objective/entropy": -83.75293732, "objective/non_score_reward": -1.72107732, "objective/rlhf_reward": 12.41173553, "objective/scores": 14.125, "policy/approxkl_avg": 0.00080864, "policy/clipfrac_avg": 0.01522262, "loss/policy_avg": -0.1442204, "loss/value_avg": 0.68889111, "val/clipfrac_avg": 0.23640706, "policy/entropy_avg": 0.14866734, "val/ratio": 0.99115521, "val/ratio_var": 5.66e-05, "val/num_eos_tokens": 388, "lr": 6.97e-06, "episode": 7984, "epoch": 0.40323232, "global_step/max_steps": "499/1238", "percentage": "40.31%", "elapsed_time": "5h 59m 42s", "remaining_time": "8h 52m 42s"} +{"eps": 0, "objective/kl": 28.21258545, "objective/entropy": -83.42674255, "objective/non_score_reward": -1.41062927, "objective/rlhf_reward": 12.11280823, "objective/scores": 13.5, "policy/approxkl_avg": 0.00099758, "policy/clipfrac_avg": 0.01080702, "loss/policy_avg": -0.20510384, "loss/value_avg": 0.70082772, "val/clipfrac_avg": 0.08799946, "policy/entropy_avg": 0.15832973, "val/ratio": 0.98988116, "val/ratio_var": 5.4e-05, "val/num_eos_tokens": 98, "lr": 6.96e-06, "episode": 8000, "epoch": 0.4040404, "global_step/max_steps": "500/1238", "percentage": "40.39%", "elapsed_time": "6h 0m 19s", "remaining_time": "8h 51m 50s"} +{"eps": 0, "objective/kl": 33.63490295, "objective/entropy": -98.09926605, "objective/non_score_reward": -1.68174529, "objective/rlhf_reward": 11.16981792, "objective/scores": 12.875, "policy/approxkl_avg": 0.00096161, "policy/clipfrac_avg": 0.00802622, "loss/policy_avg": 0.11570744, "loss/value_avg": 1.28279662, "val/clipfrac_avg": 0.17985009, "policy/entropy_avg": 0.16532373, "val/ratio": 0.98884994, "val/ratio_var": 0.00011337, "val/num_eos_tokens": 104, "lr": 6.95e-06, "episode": 8016, "epoch": 0.40484848, "global_step/max_steps": "501/1238", "percentage": "40.47%", "elapsed_time": "6h 1m 14s", "remaining_time": "8h 51m 24s"} +{"eps": 0, "objective/kl": 36.50762177, "objective/entropy": -92.84262848, "objective/non_score_reward": -1.82538116, "objective/rlhf_reward": 11.55743217, "objective/scores": 13.375, "policy/approxkl_avg": 0.00085737, "policy/clipfrac_avg": 0.00826902, "loss/policy_avg": 0.33835945, "loss/value_avg": 0.47949988, "val/clipfrac_avg": 0.02156223, "policy/entropy_avg": 0.14825702, "val/ratio": 0.99062216, "val/ratio_var": 2.254e-05, "val/num_eos_tokens": 38, "lr": 6.94e-06, "episode": 8032, "epoch": 0.40565657, "global_step/max_steps": "502/1238", "percentage": "40.55%", "elapsed_time": "6h 2m 18s", "remaining_time": "8h 51m 11s"} +{"eps": 0, "objective/kl": 42.31287766, "objective/entropy": -211.23294067, "objective/non_score_reward": -2.11564398, "objective/rlhf_reward": 8.3335743, "objective/scores": 10.4375, "policy/approxkl_avg": 0.00093279, "policy/clipfrac_avg": 0.00978491, "loss/policy_avg": -0.00360492, "loss/value_avg": 0.28834862, "val/clipfrac_avg": 0.08479396, "policy/entropy_avg": 0.16595292, "val/ratio": 0.98854417, "val/ratio_var": 8.162e-05, "val/num_eos_tokens": 600, "lr": 6.93e-06, "episode": 8048, "epoch": 0.40646465, "global_step/max_steps": "503/1238", "percentage": "40.63%", "elapsed_time": "6h 4m 4s", "remaining_time": "8h 51m 59s"} +{"eps": 0, "objective/kl": 26.93456268, "objective/entropy": -71.5333252, "objective/non_score_reward": -1.34672809, "objective/rlhf_reward": 11.85639668, "objective/scores": 13.1875, "policy/approxkl_avg": 0.00101302, "policy/clipfrac_avg": 0.01617375, "loss/policy_avg": 0.34567261, "loss/value_avg": 0.74628592, "val/clipfrac_avg": 0.06575201, "policy/entropy_avg": 0.18730783, "val/ratio": 0.9880873, "val/ratio_var": 9.105e-05, "val/num_eos_tokens": 245, "lr": 6.91e-06, "episode": 8064, "epoch": 0.40727273, "global_step/max_steps": "504/1238", "percentage": "40.71%", "elapsed_time": "6h 4m 39s", "remaining_time": "8h 51m 4s"} +{"eps": 0, "objective/kl": 35.24448013, "objective/entropy": -105.62996674, "objective/non_score_reward": -1.76222396, "objective/rlhf_reward": 11.6987133, "objective/scores": 13.4375, "policy/approxkl_avg": 0.00077422, "policy/clipfrac_avg": 0.01576562, "loss/policy_avg": 0.50404251, "loss/value_avg": 0.56508839, "val/clipfrac_avg": 0.04690375, "policy/entropy_avg": 0.18837833, "val/ratio": 0.99075222, "val/ratio_var": 3.814e-05, "val/num_eos_tokens": 570, "lr": 6.9e-06, "episode": 8080, "epoch": 0.40808081, "global_step/max_steps": "505/1238", "percentage": "40.79%", "elapsed_time": "6h 5m 14s", "remaining_time": "8h 50m 8s"} +{"eps": 0, "objective/kl": 27.57551956, "objective/entropy": -108.91736603, "objective/non_score_reward": -1.37877595, "objective/rlhf_reward": 12.50403595, "objective/scores": 13.875, "policy/approxkl_avg": 0.00156736, "policy/clipfrac_avg": 0.01719318, "loss/policy_avg": -0.13703673, "loss/value_avg": 1.18259287, "val/clipfrac_avg": 0.10861704, "policy/entropy_avg": 0.14080572, "val/ratio": 0.99179852, "val/ratio_var": 0.00015174, "val/num_eos_tokens": 979, "lr": 6.89e-06, "episode": 8096, "epoch": 0.40888889, "global_step/max_steps": "506/1238", "percentage": "40.87%", "elapsed_time": "6h 6m 4s", "remaining_time": "8h 49m 34s"} +{"eps": 0, "objective/kl": 27.5862484, "objective/entropy": -138.9822998, "objective/non_score_reward": -1.3793124, "objective/rlhf_reward": 11.39412594, "objective/scores": 12.75, "policy/approxkl_avg": 0.00064021, "policy/clipfrac_avg": 0.0124297, "loss/policy_avg": -0.49959701, "loss/value_avg": 1.29598415, "val/clipfrac_avg": 0.27246493, "policy/entropy_avg": 0.10546708, "val/ratio": 0.99386847, "val/ratio_var": 3.624e-05, "val/num_eos_tokens": 254, "lr": 6.88e-06, "episode": 8112, "epoch": 0.40969697, "global_step/max_steps": "507/1238", "percentage": "40.95%", "elapsed_time": "6h 6m 53s", "remaining_time": "8h 48m 59s"} +{"eps": 0, "objective/kl": 24.47213554, "objective/entropy": -50.94309235, "objective/non_score_reward": -1.22360682, "objective/rlhf_reward": 11.54201889, "objective/scores": 12.75, "policy/approxkl_avg": 0.00192724, "policy/clipfrac_avg": 0.02197218, "loss/policy_avg": 0.02236701, "loss/value_avg": 1.0374105, "val/clipfrac_avg": 0.12712614, "policy/entropy_avg": 0.12678146, "val/ratio": 0.99146807, "val/ratio_var": 3.564e-05, "val/num_eos_tokens": 35, "lr": 6.86e-06, "episode": 8128, "epoch": 0.41050505, "global_step/max_steps": "508/1238", "percentage": "41.03%", "elapsed_time": "6h 7m 44s", "remaining_time": "8h 48m 27s"} +{"eps": 0, "objective/kl": 32.95318222, "objective/entropy": -138.50244141, "objective/non_score_reward": -1.6476593, "objective/rlhf_reward": 11.1257782, "objective/scores": 12.75, "policy/approxkl_avg": 0.00094988, "policy/clipfrac_avg": 0.01550323, "loss/policy_avg": -0.09880562, "loss/value_avg": 0.75434899, "val/clipfrac_avg": 0.08852464, "policy/entropy_avg": 0.13935184, "val/ratio": 0.991184, "val/ratio_var": 4.138e-05, "val/num_eos_tokens": 273, "lr": 6.85e-06, "episode": 8144, "epoch": 0.41131313, "global_step/max_steps": "509/1238", "percentage": "41.11%", "elapsed_time": "6h 8m 37s", "remaining_time": "8h 47m 57s"} +{"eps": 0, "objective/kl": 32.37398529, "objective/entropy": -83.92607117, "objective/non_score_reward": -1.61869931, "objective/rlhf_reward": 9.77192593, "objective/scores": 11.375, "policy/approxkl_avg": 0.00085503, "policy/clipfrac_avg": 0.00683262, "loss/policy_avg": 0.0414894, "loss/value_avg": 0.46354362, "val/clipfrac_avg": 0.16616604, "policy/entropy_avg": 0.14737749, "val/ratio": 0.99004996, "val/ratio_var": 5.551e-05, "val/num_eos_tokens": 115, "lr": 6.84e-06, "episode": 8160, "epoch": 0.41212121, "global_step/max_steps": "510/1238", "percentage": "41.20%", "elapsed_time": "6h 9m 11s", "remaining_time": "8h 46m 59s"} +{"eps": 0, "objective/kl": 31.99300385, "objective/entropy": -101.5657196, "objective/non_score_reward": -1.59965014, "objective/rlhf_reward": 11.55659962, "objective/scores": 13.125, "policy/approxkl_avg": 0.00080884, "policy/clipfrac_avg": 0.01457807, "loss/policy_avg": -0.35513175, "loss/value_avg": 0.91696167, "val/clipfrac_avg": 0.14982203, "policy/entropy_avg": 0.16936588, "val/ratio": 0.9902457, "val/ratio_var": 6.044e-05, "val/num_eos_tokens": 126, "lr": 6.83e-06, "episode": 8176, "epoch": 0.41292929, "global_step/max_steps": "511/1238", "percentage": "41.28%", "elapsed_time": "6h 9m 55s", "remaining_time": "8h 46m 18s"} +{"eps": 0, "objective/kl": 28.13634872, "objective/entropy": -98.40760803, "objective/non_score_reward": -1.40681756, "objective/rlhf_reward": 10.44083786, "objective/scores": 11.875, "policy/approxkl_avg": 0.00084922, "policy/clipfrac_avg": 0.00840041, "loss/policy_avg": 0.00459339, "loss/value_avg": 0.6813367, "val/clipfrac_avg": 0.08613656, "policy/entropy_avg": 0.172279, "val/ratio": 0.9898752, "val/ratio_var": 4.234e-05, "val/num_eos_tokens": 52, "lr": 6.81e-06, "episode": 8192, "epoch": 0.41373737, "global_step/max_steps": "512/1238", "percentage": "41.36%", "elapsed_time": "6h 11m 2s", "remaining_time": "8h 46m 8s"} +{"eps": 0, "objective/kl": 32.61389923, "objective/entropy": -121.5741272, "objective/non_score_reward": -1.63069499, "objective/rlhf_reward": 10.65055466, "objective/scores": 12.25, "policy/approxkl_avg": 0.00076192, "policy/clipfrac_avg": 0.00824713, "loss/policy_avg": 0.0969256, "loss/value_avg": 0.45964065, "val/clipfrac_avg": 0.13378626, "policy/entropy_avg": 0.12896538, "val/ratio": 0.99140263, "val/ratio_var": 6.938e-05, "val/num_eos_tokens": 9, "lr": 6.8e-06, "episode": 8208, "epoch": 0.41454545, "global_step/max_steps": "513/1238", "percentage": "41.44%", "elapsed_time": "6h 12m 31s", "remaining_time": "8h 46m 28s"} +{"eps": 0, "objective/kl": 32.9668808, "objective/entropy": -133.54974365, "objective/non_score_reward": -1.64834404, "objective/rlhf_reward": 13.60165596, "objective/scores": 15.25, "policy/approxkl_avg": 0.00083729, "policy/clipfrac_avg": 0.01306083, "loss/policy_avg": -0.16650215, "loss/value_avg": 0.5488894, "val/clipfrac_avg": 0.20459802, "policy/entropy_avg": 0.11695766, "val/ratio": 0.99146593, "val/ratio_var": 3.493e-05, "val/num_eos_tokens": 21, "lr": 6.79e-06, "episode": 8224, "epoch": 0.41535354, "global_step/max_steps": "514/1238", "percentage": "41.52%", "elapsed_time": "6h 13m 22s", "remaining_time": "8h 45m 54s"} +{"eps": 0, "objective/kl": 31.56642532, "objective/entropy": -105.05879211, "objective/non_score_reward": -1.57832134, "objective/rlhf_reward": 10.85136604, "objective/scores": 12.4375, "policy/approxkl_avg": 0.00091352, "policy/clipfrac_avg": 0.01163512, "loss/policy_avg": 0.02540591, "loss/value_avg": 0.45760772, "val/clipfrac_avg": 0.12827221, "policy/entropy_avg": 0.16112518, "val/ratio": 0.98876977, "val/ratio_var": 4.425e-05, "val/num_eos_tokens": 416, "lr": 6.78e-06, "episode": 8240, "epoch": 0.41616162, "global_step/max_steps": "515/1238", "percentage": "41.60%", "elapsed_time": "6h 14m 10s", "remaining_time": "8h 45m 17s"} +{"eps": 0, "objective/kl": 32.33205032, "objective/entropy": -104.26789856, "objective/non_score_reward": -1.61660242, "objective/rlhf_reward": 10.48496056, "objective/scores": 12.125, "policy/approxkl_avg": 0.00109462, "policy/clipfrac_avg": 0.00962565, "loss/policy_avg": 0.0503514, "loss/value_avg": 0.58904392, "val/clipfrac_avg": 0.11179495, "policy/entropy_avg": 0.16122532, "val/ratio": 0.98999393, "val/ratio_var": 6.829e-05, "val/num_eos_tokens": 386, "lr": 6.76e-06, "episode": 8256, "epoch": 0.4169697, "global_step/max_steps": "516/1238", "percentage": "41.68%", "elapsed_time": "6h 14m 54s", "remaining_time": "8h 44m 35s"} +{"eps": 0, "objective/kl": 33.56115341, "objective/entropy": -76.06773376, "objective/non_score_reward": -1.67805767, "objective/rlhf_reward": 11.60319328, "objective/scores": 13.25, "policy/approxkl_avg": 0.00097492, "policy/clipfrac_avg": 0.01815664, "loss/policy_avg": 0.01639606, "loss/value_avg": 2.60082746, "val/clipfrac_avg": 0.12106802, "policy/entropy_avg": 0.19376278, "val/ratio": 0.98801649, "val/ratio_var": 5.697e-05, "val/num_eos_tokens": 163, "lr": 6.75e-06, "episode": 8272, "epoch": 0.41777778, "global_step/max_steps": "517/1238", "percentage": "41.76%", "elapsed_time": "6h 15m 35s", "remaining_time": "8h 43m 47s"} +{"eps": 0, "objective/kl": 27.76697159, "objective/entropy": -17.73412895, "objective/non_score_reward": -1.38834858, "objective/rlhf_reward": 12.83040142, "objective/scores": 14.25, "policy/approxkl_avg": 0.00087116, "policy/clipfrac_avg": 0.00449907, "loss/policy_avg": -0.07019297, "loss/value_avg": 0.5070563, "val/clipfrac_avg": 0.12934029, "policy/entropy_avg": 0.16478729, "val/ratio": 0.98892963, "val/ratio_var": 3.157e-05, "val/num_eos_tokens": 145, "lr": 6.74e-06, "episode": 8288, "epoch": 0.41858586, "global_step/max_steps": "518/1238", "percentage": "41.84%", "elapsed_time": "6h 16m 14s", "remaining_time": "8h 42m 58s"} +{"eps": 0, "objective/kl": 31.43227005, "objective/entropy": -62.78154373, "objective/non_score_reward": -1.57161331, "objective/rlhf_reward": 12.31119919, "objective/scores": 13.875, "policy/approxkl_avg": 0.00081075, "policy/clipfrac_avg": 0.00970063, "loss/policy_avg": -0.11896735, "loss/value_avg": 0.35987717, "val/clipfrac_avg": 0.12159739, "policy/entropy_avg": 0.12371361, "val/ratio": 0.99068874, "val/ratio_var": 3.533e-05, "val/num_eos_tokens": 365, "lr": 6.73e-06, "episode": 8304, "epoch": 0.41939394, "global_step/max_steps": "519/1238", "percentage": "41.92%", "elapsed_time": "6h 16m 44s", "remaining_time": "8h 41m 55s"} +{"eps": 0, "objective/kl": 32.71669769, "objective/entropy": -86.20362854, "objective/non_score_reward": -1.63583493, "objective/rlhf_reward": 12.60635185, "objective/scores": 14.25, "policy/approxkl_avg": 0.0021236, "policy/clipfrac_avg": 0.01577186, "loss/policy_avg": -0.07763407, "loss/value_avg": 0.31349975, "val/clipfrac_avg": 0.05747884, "policy/entropy_avg": 0.16243172, "val/ratio": 0.9893775, "val/ratio_var": 5.578e-05, "val/num_eos_tokens": 30, "lr": 6.71e-06, "episode": 8320, "epoch": 0.42020202, "global_step/max_steps": "520/1238", "percentage": "42.00%", "elapsed_time": "6h 17m 22s", "remaining_time": "8h 41m 4s"} +{"eps": 0, "objective/kl": 23.35737228, "objective/entropy": -131.93936157, "objective/non_score_reward": -1.16786861, "objective/rlhf_reward": 12.73838043, "objective/scores": 13.875, "policy/approxkl_avg": 0.00083313, "policy/clipfrac_avg": 0.01891532, "loss/policy_avg": -0.464477, "loss/value_avg": 1.10380161, "val/clipfrac_avg": 0.16148424, "policy/entropy_avg": 0.14388728, "val/ratio": 0.99084604, "val/ratio_var": 6.456e-05, "val/num_eos_tokens": 377, "lr": 6.7e-06, "episode": 8336, "epoch": 0.4210101, "global_step/max_steps": "521/1238", "percentage": "42.08%", "elapsed_time": "6h 17m 55s", "remaining_time": "8h 40m 6s"} +{"eps": 0, "objective/kl": 27.40955353, "objective/entropy": -114.08953094, "objective/non_score_reward": -1.37047768, "objective/rlhf_reward": 11.65295982, "objective/scores": 13.0, "policy/approxkl_avg": 0.00085416, "policy/clipfrac_avg": 0.01524645, "loss/policy_avg": -0.13082063, "loss/value_avg": 0.6008985, "val/clipfrac_avg": 0.08382975, "policy/entropy_avg": 0.15891838, "val/ratio": 0.99124432, "val/ratio_var": 5.435e-05, "val/num_eos_tokens": 337, "lr": 6.69e-06, "episode": 8352, "epoch": 0.42181818, "global_step/max_steps": "522/1238", "percentage": "42.16%", "elapsed_time": "6h 18m 28s", "remaining_time": "8h 39m 7s"} +{"eps": 0, "objective/kl": 34.11695099, "objective/entropy": -129.62985229, "objective/non_score_reward": -1.70584762, "objective/rlhf_reward": 10.25118446, "objective/scores": 11.9375, "policy/approxkl_avg": 0.0007716, "policy/clipfrac_avg": 0.00590224, "loss/policy_avg": -0.14331807, "loss/value_avg": 0.31795254, "val/clipfrac_avg": 0.14383307, "policy/entropy_avg": 0.14097148, "val/ratio": 0.99052471, "val/ratio_var": 4.685e-05, "val/num_eos_tokens": 34, "lr": 6.68e-06, "episode": 8368, "epoch": 0.42262626, "global_step/max_steps": "523/1238", "percentage": "42.25%", "elapsed_time": "6h 19m 14s", "remaining_time": "8h 38m 28s"} +{"eps": 0, "objective/kl": 30.30418396, "objective/entropy": -46.01490784, "objective/non_score_reward": -1.5152092, "objective/rlhf_reward": 13.3754158, "objective/scores": 14.875, "policy/approxkl_avg": 0.00097434, "policy/clipfrac_avg": 0.00850856, "loss/policy_avg": 0.09673984, "loss/value_avg": 0.37219128, "val/clipfrac_avg": 0.07922306, "policy/entropy_avg": 0.19330502, "val/ratio": 0.98899543, "val/ratio_var": 0.00010106, "val/num_eos_tokens": 4, "lr": 6.66e-06, "episode": 8384, "epoch": 0.42343434, "global_step/max_steps": "524/1238", "percentage": "42.33%", "elapsed_time": "6h 19m 52s", "remaining_time": "8h 37m 36s"} +{"eps": 0, "objective/kl": 26.1517601, "objective/entropy": -73.01426697, "objective/non_score_reward": -1.30758798, "objective/rlhf_reward": 12.48147392, "objective/scores": 13.8125, "policy/approxkl_avg": 0.000894, "policy/clipfrac_avg": 0.01135739, "loss/policy_avg": 0.42921364, "loss/value_avg": 0.41969559, "val/clipfrac_avg": 0.04973672, "policy/entropy_avg": 0.18736029, "val/ratio": 0.99068248, "val/ratio_var": 4.821e-05, "val/num_eos_tokens": 32, "lr": 6.65e-06, "episode": 8400, "epoch": 0.42424242, "global_step/max_steps": "525/1238", "percentage": "42.41%", "elapsed_time": "6h 20m 26s", "remaining_time": "8h 36m 39s"} +{"eps": 0, "objective/kl": 29.63160515, "objective/entropy": -122.70181274, "objective/non_score_reward": -1.48158026, "objective/rlhf_reward": 10.88560677, "objective/scores": 12.375, "policy/approxkl_avg": 0.00086365, "policy/clipfrac_avg": 0.01251272, "loss/policy_avg": 0.1258349, "loss/value_avg": 0.46568286, "val/clipfrac_avg": 0.0325885, "policy/entropy_avg": 0.16792959, "val/ratio": 0.99031413, "val/ratio_var": 3.828e-05, "val/num_eos_tokens": 132, "lr": 6.64e-06, "episode": 8416, "epoch": 0.42505051, "global_step/max_steps": "526/1238", "percentage": "42.49%", "elapsed_time": "6h 21m 8s", "remaining_time": "8h 35m 54s"} +{"eps": 0, "objective/kl": 26.33936119, "objective/entropy": -157.23638916, "objective/non_score_reward": -1.31696796, "objective/rlhf_reward": 12.40959358, "objective/scores": 13.75, "policy/approxkl_avg": 0.00097356, "policy/clipfrac_avg": 0.00733454, "loss/policy_avg": 0.07225545, "loss/value_avg": 1.49367607, "val/clipfrac_avg": 0.0877819, "policy/entropy_avg": 0.12781143, "val/ratio": 0.99315107, "val/ratio_var": 0.00014699, "val/num_eos_tokens": 42, "lr": 6.63e-06, "episode": 8432, "epoch": 0.42585859, "global_step/max_steps": "527/1238", "percentage": "42.57%", "elapsed_time": "6h 22m 9s", "remaining_time": "8h 35m 35s"} +{"eps": 0, "objective/kl": 39.47442627, "objective/entropy": -100.36758423, "objective/non_score_reward": -1.97372127, "objective/rlhf_reward": 10.36221695, "objective/scores": 12.3125, "policy/approxkl_avg": 0.00092998, "policy/clipfrac_avg": 0.01164168, "loss/policy_avg": 0.2405272, "loss/value_avg": 0.3002817, "val/clipfrac_avg": 0.17108478, "policy/entropy_avg": 0.1696496, "val/ratio": 0.9882971, "val/ratio_var": 5.372e-05, "val/num_eos_tokens": 247, "lr": 6.61e-06, "episode": 8448, "epoch": 0.42666667, "global_step/max_steps": "528/1238", "percentage": "42.65%", "elapsed_time": "6h 22m 52s", "remaining_time": "8h 34m 51s"} +{"eps": 0, "objective/kl": 30.68231964, "objective/entropy": -314.84899902, "objective/non_score_reward": -1.53411615, "objective/rlhf_reward": 11.43463326, "objective/scores": 13.0, "policy/approxkl_avg": 0.00096988, "policy/clipfrac_avg": 0.01093447, "loss/policy_avg": -0.15250246, "loss/value_avg": 0.87148905, "val/clipfrac_avg": 0.09076749, "policy/entropy_avg": 0.10747766, "val/ratio": 0.99223757, "val/ratio_var": 6.258e-05, "val/num_eos_tokens": 650, "lr": 6.6e-06, "episode": 8464, "epoch": 0.42747475, "global_step/max_steps": "529/1238", "percentage": "42.73%", "elapsed_time": "6h 24m 33s", "remaining_time": "8h 35m 24s"} +{"eps": 0, "objective/kl": 41.18302917, "objective/entropy": -109.72494507, "objective/non_score_reward": -2.05915165, "objective/rlhf_reward": 11.9330368, "objective/scores": 14.0, "policy/approxkl_avg": 0.00077082, "policy/clipfrac_avg": 0.00590625, "loss/policy_avg": -0.35478073, "loss/value_avg": 0.80448532, "val/clipfrac_avg": 0.06992866, "policy/entropy_avg": 0.14908886, "val/ratio": 0.99033332, "val/ratio_var": 7.136e-05, "val/num_eos_tokens": 571, "lr": 6.59e-06, "episode": 8480, "epoch": 0.42828283, "global_step/max_steps": "530/1238", "percentage": "42.81%", "elapsed_time": "6h 25m 8s", "remaining_time": "8h 34m 29s"} +{"eps": 0, "objective/kl": 32.64395523, "objective/entropy": -76.89524841, "objective/non_score_reward": -1.63219798, "objective/rlhf_reward": 10.93420792, "objective/scores": 12.5625, "policy/approxkl_avg": 0.00099065, "policy/clipfrac_avg": 0.0066182, "loss/policy_avg": 0.08454768, "loss/value_avg": 0.41387892, "val/clipfrac_avg": 0.08779165, "policy/entropy_avg": 0.17997742, "val/ratio": 0.98770607, "val/ratio_var": 5.024e-05, "val/num_eos_tokens": 947, "lr": 6.58e-06, "episode": 8496, "epoch": 0.42909091, "global_step/max_steps": "531/1238", "percentage": "42.89%", "elapsed_time": "6h 25m 59s", "remaining_time": "8h 33m 55s"} +{"eps": 0, "objective/kl": 31.94013596, "objective/entropy": -62.3012886, "objective/non_score_reward": -1.59700692, "objective/rlhf_reward": 12.58268166, "objective/scores": 14.1875, "policy/approxkl_avg": 0.00103268, "policy/clipfrac_avg": 0.00547721, "loss/policy_avg": -0.68562311, "loss/value_avg": 0.97275418, "val/clipfrac_avg": 0.19986156, "policy/entropy_avg": 0.17784578, "val/ratio": 0.98810202, "val/ratio_var": 5.74e-05, "val/num_eos_tokens": 38, "lr": 6.56e-06, "episode": 8512, "epoch": 0.42989899, "global_step/max_steps": "532/1238", "percentage": "42.97%", "elapsed_time": "6h 26m 34s", "remaining_time": "8h 33m 0s"} +{"eps": 0, "objective/kl": 24.37988091, "objective/entropy": -143.62107849, "objective/non_score_reward": -1.21899414, "objective/rlhf_reward": 12.21069336, "objective/scores": 13.4375, "policy/approxkl_avg": 0.00084402, "policy/clipfrac_avg": 0.00961468, "loss/policy_avg": -0.00426415, "loss/value_avg": 0.30976307, "val/clipfrac_avg": 0.0525171, "policy/entropy_avg": 0.13739681, "val/ratio": 0.99006963, "val/ratio_var": 6.144e-05, "val/num_eos_tokens": 10, "lr": 6.55e-06, "episode": 8528, "epoch": 0.43070707, "global_step/max_steps": "533/1238", "percentage": "43.05%", "elapsed_time": "6h 27m 30s", "remaining_time": "8h 32m 32s"} +{"eps": 0, "objective/kl": 30.96167946, "objective/entropy": -39.87112427, "objective/non_score_reward": -1.54808402, "objective/rlhf_reward": 13.41285324, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00118389, "policy/clipfrac_avg": 0.01087413, "loss/policy_avg": -0.09833643, "loss/value_avg": 0.35591847, "val/clipfrac_avg": 0.11940992, "policy/entropy_avg": 0.18686485, "val/ratio": 0.98730689, "val/ratio_var": 5.543e-05, "val/num_eos_tokens": 60, "lr": 6.54e-06, "episode": 8544, "epoch": 0.43151515, "global_step/max_steps": "534/1238", "percentage": "43.13%", "elapsed_time": "6h 28m 8s", "remaining_time": "8h 31m 41s"} +{"eps": 0, "objective/kl": 39.48604584, "objective/entropy": -58.2779274, "objective/non_score_reward": -1.97430229, "objective/rlhf_reward": 10.91632271, "objective/scores": 12.875, "policy/approxkl_avg": 0.00086732, "policy/clipfrac_avg": 0.00511399, "loss/policy_avg": 0.18861787, "loss/value_avg": 0.37097919, "val/clipfrac_avg": 0.19597015, "policy/entropy_avg": 0.16251945, "val/ratio": 0.98912227, "val/ratio_var": 2.865e-05, "val/num_eos_tokens": 189, "lr": 6.52e-06, "episode": 8560, "epoch": 0.43232323, "global_step/max_steps": "535/1238", "percentage": "43.21%", "elapsed_time": "6h 28m 53s", "remaining_time": "8h 31m 0s"} +{"eps": 0, "objective/kl": 28.33976364, "objective/entropy": -41.29559326, "objective/non_score_reward": -1.41698825, "objective/rlhf_reward": 12.31738663, "objective/scores": 13.75, "policy/approxkl_avg": 0.00079944, "policy/clipfrac_avg": 0.0173538, "loss/policy_avg": -0.03097318, "loss/value_avg": 0.61042845, "val/clipfrac_avg": 0.06199117, "policy/entropy_avg": 0.17484283, "val/ratio": 0.99002099, "val/ratio_var": 5.227e-05, "val/num_eos_tokens": 158, "lr": 6.51e-06, "episode": 8576, "epoch": 0.43313131, "global_step/max_steps": "536/1238", "percentage": "43.30%", "elapsed_time": "6h 29m 25s", "remaining_time": "8h 30m 1s"} +{"eps": 0, "objective/kl": 36.66104889, "objective/entropy": -98.15419769, "objective/non_score_reward": -1.8330524, "objective/rlhf_reward": 11.97163486, "objective/scores": 13.8125, "policy/approxkl_avg": 0.00111004, "policy/clipfrac_avg": 0.00995898, "loss/policy_avg": -0.14375049, "loss/value_avg": 0.18352829, "val/clipfrac_avg": 0.05129722, "policy/entropy_avg": 0.14308548, "val/ratio": 0.99019051, "val/ratio_var": 3.793e-05, "val/num_eos_tokens": 479, "lr": 6.5e-06, "episode": 8592, "epoch": 0.43393939, "global_step/max_steps": "537/1238", "percentage": "43.38%", "elapsed_time": "6h 30m 33s", "remaining_time": "8h 29m 49s"} +{"eps": 0, "objective/kl": 31.6731987, "objective/entropy": -113.62753296, "objective/non_score_reward": -1.58365989, "objective/rlhf_reward": 12.43196487, "objective/scores": 14.0, "policy/approxkl_avg": 0.00143907, "policy/clipfrac_avg": 0.02696001, "loss/policy_avg": -0.06621084, "loss/value_avg": 0.62155575, "val/clipfrac_avg": 0.1071077, "policy/entropy_avg": 0.13946056, "val/ratio": 0.98960733, "val/ratio_var": 9.375e-05, "val/num_eos_tokens": 342, "lr": 6.49e-06, "episode": 8608, "epoch": 0.43474747, "global_step/max_steps": "538/1238", "percentage": "43.46%", "elapsed_time": "6h 31m 16s", "remaining_time": "8h 29m 5s"} +{"eps": 0, "objective/kl": 34.75250244, "objective/entropy": -120.77999878, "objective/non_score_reward": -1.73762512, "objective/rlhf_reward": 11.76237488, "objective/scores": 13.5, "policy/approxkl_avg": 0.00091487, "policy/clipfrac_avg": 0.00517404, "loss/policy_avg": -0.06200865, "loss/value_avg": 0.35541371, "val/clipfrac_avg": 0.0683419, "policy/entropy_avg": 0.142416, "val/ratio": 0.989712, "val/ratio_var": 5.696e-05, "val/num_eos_tokens": 484, "lr": 6.47e-06, "episode": 8624, "epoch": 0.43555556, "global_step/max_steps": "539/1238", "percentage": "43.54%", "elapsed_time": "6h 32m 1s", "remaining_time": "8h 28m 24s"} +{"eps": 0, "objective/kl": 40.74346161, "objective/entropy": -115.64312744, "objective/non_score_reward": -2.03717303, "objective/rlhf_reward": 10.94720268, "objective/scores": 13.0, "policy/approxkl_avg": 0.00080064, "policy/clipfrac_avg": 0.00654954, "loss/policy_avg": 0.06098372, "loss/value_avg": 0.47105753, "val/clipfrac_avg": 0.15336186, "policy/entropy_avg": 0.14135742, "val/ratio": 0.99157482, "val/ratio_var": 5.314e-05, "val/num_eos_tokens": 624, "lr": 6.46e-06, "episode": 8640, "epoch": 0.43636364, "global_step/max_steps": "540/1238", "percentage": "43.62%", "elapsed_time": "6h 32m 44s", "remaining_time": "8h 27m 38s"} +{"eps": 0, "objective/kl": 31.47679901, "objective/entropy": -112.73287964, "objective/non_score_reward": -1.57384014, "objective/rlhf_reward": 13.25428486, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00093827, "policy/clipfrac_avg": 0.00647447, "loss/policy_avg": -0.39746806, "loss/value_avg": 1.79294813, "val/clipfrac_avg": 0.23907298, "policy/entropy_avg": 0.15688324, "val/ratio": 0.99058938, "val/ratio_var": 9.63e-05, "val/num_eos_tokens": 1255, "lr": 6.45e-06, "episode": 8656, "epoch": 0.43717172, "global_step/max_steps": "541/1238", "percentage": "43.70%", "elapsed_time": "6h 33m 48s", "remaining_time": "8h 27m 22s"} +{"eps": 0, "objective/kl": 33.55923462, "objective/entropy": -104.7766037, "objective/non_score_reward": -1.67796183, "objective/rlhf_reward": 13.17360115, "objective/scores": 14.875, "policy/approxkl_avg": 0.00106528, "policy/clipfrac_avg": 0.00572814, "loss/policy_avg": -0.20038423, "loss/value_avg": 0.42031449, "val/clipfrac_avg": 0.13558963, "policy/entropy_avg": 0.16969299, "val/ratio": 0.9887448, "val/ratio_var": 5.706e-05, "val/num_eos_tokens": 1683, "lr": 6.44e-06, "episode": 8672, "epoch": 0.4379798, "global_step/max_steps": "542/1238", "percentage": "43.78%", "elapsed_time": "6h 35m 19s", "remaining_time": "8h 27m 39s"} +{"eps": 0, "objective/kl": 29.88462067, "objective/entropy": -78.30276489, "objective/non_score_reward": -1.49423099, "objective/rlhf_reward": 13.05264378, "objective/scores": 14.5625, "policy/approxkl_avg": 0.00088321, "policy/clipfrac_avg": 0.01044277, "loss/policy_avg": -0.05463821, "loss/value_avg": 0.3389419, "val/clipfrac_avg": 0.05107077, "policy/entropy_avg": 0.14591885, "val/ratio": 0.9910211, "val/ratio_var": 3.291e-05, "val/num_eos_tokens": 112, "lr": 6.42e-06, "episode": 8688, "epoch": 0.43878788, "global_step/max_steps": "543/1238", "percentage": "43.86%", "elapsed_time": "6h 36m 1s", "remaining_time": "8h 26m 52s"} +{"eps": 0, "objective/kl": 41.26933289, "objective/entropy": -48.91859818, "objective/non_score_reward": -2.06346679, "objective/rlhf_reward": 12.56153297, "objective/scores": 14.625, "policy/approxkl_avg": 0.00084299, "policy/clipfrac_avg": 0.00608425, "loss/policy_avg": 0.16157524, "loss/value_avg": 0.4397819, "val/clipfrac_avg": 0.052078, "policy/entropy_avg": 0.15523911, "val/ratio": 0.99016619, "val/ratio_var": 4.473e-05, "val/num_eos_tokens": 150, "lr": 6.41e-06, "episode": 8704, "epoch": 0.43959596, "global_step/max_steps": "544/1238", "percentage": "43.94%", "elapsed_time": "6h 37m 4s", "remaining_time": "8h 26m 33s"} +{"eps": 0, "objective/kl": 35.15542603, "objective/entropy": -105.18928528, "objective/non_score_reward": -1.75777125, "objective/rlhf_reward": 14.87504196, "objective/scores": 16.625, "policy/approxkl_avg": 0.00110772, "policy/clipfrac_avg": 0.01006338, "loss/policy_avg": -0.15311198, "loss/value_avg": 0.49878266, "val/clipfrac_avg": 0.10631219, "policy/entropy_avg": 0.17020035, "val/ratio": 0.98998398, "val/ratio_var": 8e-05, "val/num_eos_tokens": 501, "lr": 6.4e-06, "episode": 8720, "epoch": 0.44040404, "global_step/max_steps": "545/1238", "percentage": "44.02%", "elapsed_time": "6h 37m 39s", "remaining_time": "8h 25m 39s"} +{"eps": 0, "objective/kl": 31.34070015, "objective/entropy": -31.19978714, "objective/non_score_reward": -1.56703508, "objective/rlhf_reward": 11.19077682, "objective/scores": 12.75, "policy/approxkl_avg": 0.00096822, "policy/clipfrac_avg": 0.00647999, "loss/policy_avg": 0.10405912, "loss/value_avg": 0.42140341, "val/clipfrac_avg": 0.10993473, "policy/entropy_avg": 0.20100021, "val/ratio": 0.98901737, "val/ratio_var": 3.694e-05, "val/num_eos_tokens": 238, "lr": 6.38e-06, "episode": 8736, "epoch": 0.44121212, "global_step/max_steps": "546/1238", "percentage": "44.10%", "elapsed_time": "6h 38m 35s", "remaining_time": "8h 25m 10s"} +{"eps": 0, "objective/kl": 34.65884781, "objective/entropy": -39.1321106, "objective/non_score_reward": -1.73294234, "objective/rlhf_reward": 9.94674492, "objective/scores": 11.6875, "policy/approxkl_avg": 0.00095225, "policy/clipfrac_avg": 0.01447022, "loss/policy_avg": 0.53591603, "loss/value_avg": 1.82511687, "val/clipfrac_avg": 0.09755296, "policy/entropy_avg": 0.18070626, "val/ratio": 0.9890222, "val/ratio_var": 4.67e-05, "val/num_eos_tokens": 147, "lr": 6.37e-06, "episode": 8752, "epoch": 0.4420202, "global_step/max_steps": "547/1238", "percentage": "44.18%", "elapsed_time": "6h 39m 2s", "remaining_time": "8h 24m 4s"} +{"eps": 0, "objective/kl": 37.90512085, "objective/entropy": -175.40753174, "objective/non_score_reward": -1.89525604, "objective/rlhf_reward": 12.60865021, "objective/scores": 14.5, "policy/approxkl_avg": 0.00083828, "policy/clipfrac_avg": 0.00394605, "loss/policy_avg": 0.00333966, "loss/value_avg": 0.30044219, "val/clipfrac_avg": 0.0730562, "policy/entropy_avg": 0.15326715, "val/ratio": 0.98982298, "val/ratio_var": 4.889e-05, "val/num_eos_tokens": 253, "lr": 6.36e-06, "episode": 8768, "epoch": 0.44282828, "global_step/max_steps": "548/1238", "percentage": "44.26%", "elapsed_time": "6h 40m 43s", "remaining_time": "8h 24m 33s"} +{"eps": 0, "objective/kl": 31.03716278, "objective/entropy": -127.72113037, "objective/non_score_reward": -1.55185807, "objective/rlhf_reward": 13.63564205, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00085822, "policy/clipfrac_avg": 0.01285835, "loss/policy_avg": 0.07848239, "loss/value_avg": 1.09365249, "val/clipfrac_avg": 0.10532165, "policy/entropy_avg": 0.17509651, "val/ratio": 0.9902795, "val/ratio_var": 5.98e-05, "val/num_eos_tokens": 707, "lr": 6.35e-06, "episode": 8784, "epoch": 0.44363636, "global_step/max_steps": "549/1238", "percentage": "44.35%", "elapsed_time": "6h 41m 23s", "remaining_time": "8h 23m 45s"} +{"eps": 0, "objective/kl": 30.36469269, "objective/entropy": -142.31295776, "objective/non_score_reward": -1.51823473, "objective/rlhf_reward": 13.25520325, "objective/scores": 14.75, "policy/approxkl_avg": 0.00093788, "policy/clipfrac_avg": 0.00806362, "loss/policy_avg": -0.13267037, "loss/value_avg": 0.69792008, "val/clipfrac_avg": 0.21628013, "policy/entropy_avg": 0.1446209, "val/ratio": 0.98997498, "val/ratio_var": 7.446e-05, "val/num_eos_tokens": 1703, "lr": 6.33e-06, "episode": 8800, "epoch": 0.44444444, "global_step/max_steps": "550/1238", "percentage": "44.43%", "elapsed_time": "6h 43m 9s", "remaining_time": "8h 24m 19s"} +{"eps": 0, "objective/kl": 28.54915047, "objective/entropy": -74.91482544, "objective/non_score_reward": -1.42745745, "objective/rlhf_reward": 13.55691719, "objective/scores": 15.0, "policy/approxkl_avg": 0.00111125, "policy/clipfrac_avg": 0.01159308, "loss/policy_avg": 0.15575792, "loss/value_avg": 0.44957703, "val/clipfrac_avg": 0.09938866, "policy/entropy_avg": 0.17023122, "val/ratio": 0.98689419, "val/ratio_var": 0.00012118, "val/num_eos_tokens": 124, "lr": 6.32e-06, "episode": 8816, "epoch": 0.44525253, "global_step/max_steps": "551/1238", "percentage": "44.51%", "elapsed_time": "6h 43m 53s", "remaining_time": "8h 23m 34s"} +{"eps": 0, "objective/kl": 38.20920944, "objective/entropy": -101.52589417, "objective/non_score_reward": -1.91046047, "objective/rlhf_reward": 13.23016357, "objective/scores": 15.125, "policy/approxkl_avg": 0.00108056, "policy/clipfrac_avg": 0.01246523, "loss/policy_avg": -0.15809897, "loss/value_avg": 0.58488703, "val/clipfrac_avg": 0.03225298, "policy/entropy_avg": 0.18145943, "val/ratio": 0.98823327, "val/ratio_var": 5.319e-05, "val/num_eos_tokens": 138, "lr": 6.31e-06, "episode": 8832, "epoch": 0.44606061, "global_step/max_steps": "552/1238", "percentage": "44.59%", "elapsed_time": "6h 44m 37s", "remaining_time": "8h 22m 50s"} +{"eps": 0, "objective/kl": 36.2408371, "objective/entropy": -113.32449341, "objective/non_score_reward": -1.81204188, "objective/rlhf_reward": 11.74264526, "objective/scores": 13.5625, "policy/approxkl_avg": 0.00104946, "policy/clipfrac_avg": 0.0122326, "loss/policy_avg": 0.11628187, "loss/value_avg": 0.64049804, "val/clipfrac_avg": 0.14791587, "policy/entropy_avg": 0.16427016, "val/ratio": 0.98831856, "val/ratio_var": 6.705e-05, "val/num_eos_tokens": 22, "lr": 6.29e-06, "episode": 8848, "epoch": 0.44686869, "global_step/max_steps": "553/1238", "percentage": "44.67%", "elapsed_time": "6h 45m 22s", "remaining_time": "8h 22m 8s"} +{"eps": 0, "objective/kl": 36.22610092, "objective/entropy": -56.46511459, "objective/non_score_reward": -1.81130505, "objective/rlhf_reward": 10.86838245, "objective/scores": 12.6875, "policy/approxkl_avg": 0.00096099, "policy/clipfrac_avg": 0.02042119, "loss/policy_avg": 0.55289745, "loss/value_avg": 0.93374652, "val/clipfrac_avg": 0.02570946, "policy/entropy_avg": 0.21524811, "val/ratio": 0.98996514, "val/ratio_var": 2.996e-05, "val/num_eos_tokens": 5, "lr": 6.28e-06, "episode": 8864, "epoch": 0.44767677, "global_step/max_steps": "554/1238", "percentage": "44.75%", "elapsed_time": "6h 45m 48s", "remaining_time": "8h 21m 2s"} +{"eps": 0, "objective/kl": 43.3099556, "objective/entropy": -92.14622498, "objective/non_score_reward": -2.16549778, "objective/rlhf_reward": 12.53762817, "objective/scores": 14.6875, "policy/approxkl_avg": 0.00151919, "policy/clipfrac_avg": 0.00895031, "loss/policy_avg": -0.13656527, "loss/value_avg": 0.38830268, "val/clipfrac_avg": 0.12284453, "policy/entropy_avg": 0.16514254, "val/ratio": 0.98890454, "val/ratio_var": 9.022e-05, "val/num_eos_tokens": 564, "lr": 6.27e-06, "episode": 8880, "epoch": 0.44848485, "global_step/max_steps": "555/1238", "percentage": "44.83%", "elapsed_time": "6h 46m 24s", "remaining_time": "8h 20m 8s"} +{"eps": 0, "objective/kl": 42.43487549, "objective/entropy": -117.01277924, "objective/non_score_reward": -2.12174392, "objective/rlhf_reward": 12.10481834, "objective/scores": 14.25, "policy/approxkl_avg": 0.00082879, "policy/clipfrac_avg": 0.00863044, "loss/policy_avg": -0.27194309, "loss/value_avg": 0.51979172, "val/clipfrac_avg": 0.06423404, "policy/entropy_avg": 0.13199997, "val/ratio": 0.99037588, "val/ratio_var": 5.324e-05, "val/num_eos_tokens": 354, "lr": 6.26e-06, "episode": 8896, "epoch": 0.44929293, "global_step/max_steps": "556/1238", "percentage": "44.91%", "elapsed_time": "6h 46m 58s", "remaining_time": "8h 19m 12s"} +{"eps": 0, "objective/kl": 42.79301453, "objective/entropy": -73.03179932, "objective/non_score_reward": -2.13965082, "objective/rlhf_reward": 12.7587862, "objective/scores": 14.875, "policy/approxkl_avg": 0.00089026, "policy/clipfrac_avg": 0.00688001, "loss/policy_avg": -0.03566729, "loss/value_avg": 0.1839591, "val/clipfrac_avg": 0.0390281, "policy/entropy_avg": 0.16189003, "val/ratio": 0.98932505, "val/ratio_var": 4.172e-05, "val/num_eos_tokens": 239, "lr": 6.24e-06, "episode": 8912, "epoch": 0.45010101, "global_step/max_steps": "557/1238", "percentage": "44.99%", "elapsed_time": "6h 47m 33s", "remaining_time": "8h 18m 17s"} +{"eps": 0, "objective/kl": 44.56954193, "objective/entropy": -104.41824341, "objective/non_score_reward": -2.22847724, "objective/rlhf_reward": 13.10746002, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00090929, "policy/clipfrac_avg": 0.01071926, "loss/policy_avg": 0.02715391, "loss/value_avg": 0.41223347, "val/clipfrac_avg": 0.03966514, "policy/entropy_avg": 0.13871384, "val/ratio": 0.99003148, "val/ratio_var": 5.081e-05, "val/num_eos_tokens": 642, "lr": 6.23e-06, "episode": 8928, "epoch": 0.45090909, "global_step/max_steps": "558/1238", "percentage": "45.07%", "elapsed_time": "6h 48m 9s", "remaining_time": "8h 17m 23s"} +{"eps": 0, "objective/kl": 46.45120239, "objective/entropy": -57.5327301, "objective/non_score_reward": -2.32256031, "objective/rlhf_reward": 11.65400219, "objective/scores": 14.0, "policy/approxkl_avg": 0.00099851, "policy/clipfrac_avg": 0.00956519, "loss/policy_avg": 0.13781857, "loss/value_avg": 0.54256678, "val/clipfrac_avg": 0.14630571, "policy/entropy_avg": 0.16213226, "val/ratio": 0.98971331, "val/ratio_var": 4.262e-05, "val/num_eos_tokens": 76, "lr": 6.22e-06, "episode": 8944, "epoch": 0.45171717, "global_step/max_steps": "559/1238", "percentage": "45.15%", "elapsed_time": "6h 48m 40s", "remaining_time": "8h 16m 24s"} +{"eps": 0, "objective/kl": 35.97865677, "objective/entropy": -166.40460205, "objective/non_score_reward": -1.79893291, "objective/rlhf_reward": 11.09950447, "objective/scores": 12.875, "policy/approxkl_avg": 0.00148418, "policy/clipfrac_avg": 0.01165555, "loss/policy_avg": -0.00900414, "loss/value_avg": 0.3125574, "val/clipfrac_avg": 0.06238756, "policy/entropy_avg": 0.1568408, "val/ratio": 0.98961812, "val/ratio_var": 8.537e-05, "val/num_eos_tokens": 381, "lr": 6.2e-06, "episode": 8960, "epoch": 0.45252525, "global_step/max_steps": "560/1238", "percentage": "45.23%", "elapsed_time": "6h 50m 26s", "remaining_time": "8h 16m 56s"} +{"eps": 0, "objective/kl": 30.72741318, "objective/entropy": -117.21914673, "objective/non_score_reward": -1.53637064, "objective/rlhf_reward": 11.63550377, "objective/scores": 13.1875, "policy/approxkl_avg": 0.00131384, "policy/clipfrac_avg": 0.01345793, "loss/policy_avg": -0.01149874, "loss/value_avg": 0.55935937, "val/clipfrac_avg": 0.06608336, "policy/entropy_avg": 0.16321659, "val/ratio": 0.98888862, "val/ratio_var": 6.118e-05, "val/num_eos_tokens": 5, "lr": 6.19e-06, "episode": 8976, "epoch": 0.45333333, "global_step/max_steps": "561/1238", "percentage": "45.32%", "elapsed_time": "6h 52m 9s", "remaining_time": "8h 17m 22s"} +{"eps": 0, "objective/kl": 36.15485001, "objective/entropy": -31.29468155, "objective/non_score_reward": -1.8077426, "objective/rlhf_reward": 12.18444443, "objective/scores": 14.0, "policy/approxkl_avg": 0.00102739, "policy/clipfrac_avg": 0.00702068, "loss/policy_avg": -0.05104259, "loss/value_avg": 0.30408823, "val/clipfrac_avg": 0.06005094, "policy/entropy_avg": 0.17518425, "val/ratio": 0.98715335, "val/ratio_var": 6.729e-05, "val/num_eos_tokens": 162, "lr": 6.18e-06, "episode": 8992, "epoch": 0.45414141, "global_step/max_steps": "562/1238", "percentage": "45.40%", "elapsed_time": "6h 52m 42s", "remaining_time": "8h 16m 25s"} +{"eps": 0, "objective/kl": 36.54793549, "objective/entropy": -124.56060791, "objective/non_score_reward": -1.82739687, "objective/rlhf_reward": 13.20385265, "objective/scores": 15.0, "policy/approxkl_avg": 0.00089375, "policy/clipfrac_avg": 0.01175685, "loss/policy_avg": 0.03235239, "loss/value_avg": 0.70624095, "val/clipfrac_avg": 0.08029687, "policy/entropy_avg": 0.1365099, "val/ratio": 0.99195981, "val/ratio_var": 6.22e-05, "val/num_eos_tokens": 503, "lr": 6.16e-06, "episode": 9008, "epoch": 0.45494949, "global_step/max_steps": "563/1238", "percentage": "45.48%", "elapsed_time": "6h 53m 38s", "remaining_time": "8h 15m 55s"} +{"eps": 0, "objective/kl": 33.58654022, "objective/entropy": -140.72860718, "objective/non_score_reward": -1.67932713, "objective/rlhf_reward": 11.43786049, "objective/scores": 13.125, "policy/approxkl_avg": 0.00072593, "policy/clipfrac_avg": 0.00824237, "loss/policy_avg": 0.08296541, "loss/value_avg": 0.46131048, "val/clipfrac_avg": 0.04573878, "policy/entropy_avg": 0.1111002, "val/ratio": 0.99297571, "val/ratio_var": 2.962e-05, "val/num_eos_tokens": 393, "lr": 6.15e-06, "episode": 9024, "epoch": 0.45575758, "global_step/max_steps": "564/1238", "percentage": "45.56%", "elapsed_time": "6h 54m 29s", "remaining_time": "8h 15m 20s"} +{"eps": 0, "objective/kl": 38.57415771, "objective/entropy": -49.62328339, "objective/non_score_reward": -1.92870796, "objective/rlhf_reward": 14.23535538, "objective/scores": 16.125, "policy/approxkl_avg": 0.0010976, "policy/clipfrac_avg": 0.00487381, "loss/policy_avg": -0.1368407, "loss/value_avg": 0.28729388, "val/clipfrac_avg": 0.21986616, "policy/entropy_avg": 0.19704437, "val/ratio": 0.98632872, "val/ratio_var": 5.253e-05, "val/num_eos_tokens": 90, "lr": 6.14e-06, "episode": 9040, "epoch": 0.45656566, "global_step/max_steps": "565/1238", "percentage": "45.64%", "elapsed_time": "6h 54m 59s", "remaining_time": "8h 14m 18s"} +{"eps": 0, "objective/kl": 40.84555054, "objective/entropy": -66.99671936, "objective/non_score_reward": -2.04227757, "objective/rlhf_reward": 12.22334671, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009533, "policy/clipfrac_avg": 0.00895039, "loss/policy_avg": 0.15097515, "loss/value_avg": 0.8962791, "val/clipfrac_avg": 0.16976982, "policy/entropy_avg": 0.15424061, "val/ratio": 0.98964185, "val/ratio_var": 6.318e-05, "val/num_eos_tokens": 139, "lr": 6.13e-06, "episode": 9056, "epoch": 0.45737374, "global_step/max_steps": "566/1238", "percentage": "45.72%", "elapsed_time": "6h 55m 31s", "remaining_time": "8h 13m 20s"} +{"eps": 0, "objective/kl": 31.18183517, "objective/entropy": -69.26618958, "objective/non_score_reward": -1.55909169, "objective/rlhf_reward": 11.47997093, "objective/scores": 13.0625, "policy/approxkl_avg": 0.0010285, "policy/clipfrac_avg": 0.00773759, "loss/policy_avg": 0.00128796, "loss/value_avg": 0.6243962, "val/clipfrac_avg": 0.07829483, "policy/entropy_avg": 0.19235611, "val/ratio": 0.98674071, "val/ratio_var": 7.091e-05, "val/num_eos_tokens": 147, "lr": 6.11e-06, "episode": 9072, "epoch": 0.45818182, "global_step/max_steps": "567/1238", "percentage": "45.80%", "elapsed_time": "6h 56m 6s", "remaining_time": "8h 12m 25s"} +{"eps": 0, "objective/kl": 28.3650589, "objective/entropy": -65.55770874, "objective/non_score_reward": -1.41825294, "objective/rlhf_reward": 13.37080956, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00092679, "policy/clipfrac_avg": 0.01309429, "loss/policy_avg": -0.19695458, "loss/value_avg": 0.67984706, "val/clipfrac_avg": 0.1175547, "policy/entropy_avg": 0.15218353, "val/ratio": 0.99065483, "val/ratio_var": 8.349e-05, "val/num_eos_tokens": 145, "lr": 6.1e-06, "episode": 9088, "epoch": 0.4589899, "global_step/max_steps": "568/1238", "percentage": "45.88%", "elapsed_time": "6h 56m 41s", "remaining_time": "8h 11m 30s"} +{"eps": 0, "objective/kl": 25.73527336, "objective/entropy": -75.15937042, "objective/non_score_reward": -1.28676367, "objective/rlhf_reward": 12.40854836, "objective/scores": 13.6875, "policy/approxkl_avg": 0.00099192, "policy/clipfrac_avg": 0.00809398, "loss/policy_avg": 0.0335924, "loss/value_avg": 0.58416659, "val/clipfrac_avg": 0.11711118, "policy/entropy_avg": 0.15917349, "val/ratio": 0.99058211, "val/ratio_var": 5.586e-05, "val/num_eos_tokens": 421, "lr": 6.09e-06, "episode": 9104, "epoch": 0.45979798, "global_step/max_steps": "569/1238", "percentage": "45.96%", "elapsed_time": "6h 57m 7s", "remaining_time": "8h 10m 25s"} +{"eps": 0, "objective/kl": 35.65166473, "objective/entropy": -101.96681976, "objective/non_score_reward": -1.78258324, "objective/rlhf_reward": 13.00647926, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00086785, "policy/clipfrac_avg": 0.00918821, "loss/policy_avg": -0.11485969, "loss/value_avg": 0.93696332, "val/clipfrac_avg": 0.04170583, "policy/entropy_avg": 0.16264248, "val/ratio": 0.98938513, "val/ratio_var": 6.943e-05, "val/num_eos_tokens": 411, "lr": 6.07e-06, "episode": 9120, "epoch": 0.46060606, "global_step/max_steps": "570/1238", "percentage": "46.04%", "elapsed_time": "6h 57m 44s", "remaining_time": "8h 9m 34s"} +{"eps": 0, "objective/kl": 34.87888336, "objective/entropy": -29.41601562, "objective/non_score_reward": -1.74394417, "objective/rlhf_reward": 13.38105583, "objective/scores": 15.125, "policy/approxkl_avg": 0.00090208, "policy/clipfrac_avg": 0.00598104, "loss/policy_avg": 0.0352929, "loss/value_avg": 0.62906742, "val/clipfrac_avg": 0.22559202, "policy/entropy_avg": 0.17604637, "val/ratio": 0.98790026, "val/ratio_var": 6.197e-05, "val/num_eos_tokens": 197, "lr": 6.06e-06, "episode": 9136, "epoch": 0.46141414, "global_step/max_steps": "571/1238", "percentage": "46.12%", "elapsed_time": "6h 58m 14s", "remaining_time": "8h 8m 33s"} +{"eps": 0, "objective/kl": 32.74857712, "objective/entropy": -94.31534576, "objective/non_score_reward": -1.637429, "objective/rlhf_reward": 11.65163326, "objective/scores": 13.3125, "policy/approxkl_avg": 0.00090006, "policy/clipfrac_avg": 0.00632565, "loss/policy_avg": 0.07575584, "loss/value_avg": 0.48475826, "val/clipfrac_avg": 0.10016693, "policy/entropy_avg": 0.16740036, "val/ratio": 0.98898697, "val/ratio_var": 4.306e-05, "val/num_eos_tokens": 374, "lr": 6.05e-06, "episode": 9152, "epoch": 0.46222222, "global_step/max_steps": "572/1238", "percentage": "46.20%", "elapsed_time": "6h 58m 57s", "remaining_time": "8h 7m 49s"} +{"eps": 0, "objective/kl": 32.94961548, "objective/entropy": -134.76264954, "objective/non_score_reward": -1.64748096, "objective/rlhf_reward": 13.68845558, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00087995, "policy/clipfrac_avg": 0.01494959, "loss/policy_avg": 0.06133082, "loss/value_avg": 0.60956788, "val/clipfrac_avg": 0.06861273, "policy/entropy_avg": 0.12122154, "val/ratio": 0.99216676, "val/ratio_var": 3.237e-05, "val/num_eos_tokens": 434, "lr": 6.03e-06, "episode": 9168, "epoch": 0.4630303, "global_step/max_steps": "573/1238", "percentage": "46.28%", "elapsed_time": "6h 59m 39s", "remaining_time": "8h 7m 1s"} +{"eps": 0, "objective/kl": 35.46616364, "objective/entropy": -40.36777496, "objective/non_score_reward": -1.77330828, "objective/rlhf_reward": 11.5235672, "objective/scores": 13.3125, "policy/approxkl_avg": 0.00120547, "policy/clipfrac_avg": 0.00810634, "loss/policy_avg": 0.00198144, "loss/value_avg": 0.4525398, "val/clipfrac_avg": 0.13928187, "policy/entropy_avg": 0.21716499, "val/ratio": 0.9869014, "val/ratio_var": 5.975e-05, "val/num_eos_tokens": 19, "lr": 6.02e-06, "episode": 9184, "epoch": 0.46383838, "global_step/max_steps": "574/1238", "percentage": "46.37%", "elapsed_time": "7h 0m 14s", "remaining_time": "8h 6m 8s"} +{"eps": 0, "objective/kl": 33.51815796, "objective/entropy": -219.04417419, "objective/non_score_reward": -1.67590785, "objective/rlhf_reward": 12.65221691, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00092889, "policy/clipfrac_avg": 0.01412575, "loss/policy_avg": -0.14026031, "loss/value_avg": 0.27154684, "val/clipfrac_avg": 0.09028111, "policy/entropy_avg": 0.12938571, "val/ratio": 0.99095267, "val/ratio_var": 6.471e-05, "val/num_eos_tokens": 30, "lr": 6.01e-06, "episode": 9200, "epoch": 0.46464646, "global_step/max_steps": "575/1238", "percentage": "46.45%", "elapsed_time": "7h 1m 54s", "remaining_time": "8h 6m 29s"} +{"eps": 0, "objective/kl": 33.28816986, "objective/entropy": -93.06488037, "objective/non_score_reward": -1.66440845, "objective/rlhf_reward": 12.04652977, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0009613, "policy/clipfrac_avg": 0.00670017, "loss/policy_avg": 0.12112676, "loss/value_avg": 0.67671609, "val/clipfrac_avg": 0.12174968, "policy/entropy_avg": 0.17998028, "val/ratio": 0.98827356, "val/ratio_var": 8.49e-05, "val/num_eos_tokens": 34, "lr": 6e-06, "episode": 9216, "epoch": 0.46545455, "global_step/max_steps": "576/1238", "percentage": "46.53%", "elapsed_time": "7h 2m 45s", "remaining_time": "8h 5m 52s"} +{"eps": 0, "objective/kl": 28.85062981, "objective/entropy": -71.49658966, "objective/non_score_reward": -1.44253159, "objective/rlhf_reward": 12.59653091, "objective/scores": 14.0625, "policy/approxkl_avg": 0.00099727, "policy/clipfrac_avg": 0.00943694, "loss/policy_avg": -0.04195672, "loss/value_avg": 0.51069379, "val/clipfrac_avg": 0.10116868, "policy/entropy_avg": 0.17303371, "val/ratio": 0.99016875, "val/ratio_var": 5.904e-05, "val/num_eos_tokens": 21, "lr": 5.98e-06, "episode": 9232, "epoch": 0.46626263, "global_step/max_steps": "577/1238", "percentage": "46.61%", "elapsed_time": "7h 3m 19s", "remaining_time": "8h 4m 57s"} +{"eps": 0, "objective/kl": 33.9830246, "objective/entropy": -94.11358643, "objective/non_score_reward": -1.69915104, "objective/rlhf_reward": 13.88678551, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00097574, "policy/clipfrac_avg": 0.00937408, "loss/policy_avg": -0.1240682, "loss/value_avg": 0.56959188, "val/clipfrac_avg": 0.09642657, "policy/entropy_avg": 0.15090752, "val/ratio": 0.98896837, "val/ratio_var": 6.235e-05, "val/num_eos_tokens": 261, "lr": 5.97e-06, "episode": 9248, "epoch": 0.46707071, "global_step/max_steps": "578/1238", "percentage": "46.69%", "elapsed_time": "7h 3m 55s", "remaining_time": "8h 4m 4s"} +{"eps": 0, "objective/kl": 40.6615715, "objective/entropy": -60.78998566, "objective/non_score_reward": -2.03307867, "objective/rlhf_reward": 13.38098335, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00100614, "policy/clipfrac_avg": 0.00698361, "loss/policy_avg": -0.01250461, "loss/value_avg": 0.16729793, "val/clipfrac_avg": 0.14943929, "policy/entropy_avg": 0.18235397, "val/ratio": 0.98732519, "val/ratio_var": 3.271e-05, "val/num_eos_tokens": 86, "lr": 5.96e-06, "episode": 9264, "epoch": 0.46787879, "global_step/max_steps": "579/1238", "percentage": "46.77%", "elapsed_time": "7h 4m 39s", "remaining_time": "8h 3m 19s"} +{"eps": 0, "objective/kl": 44.12640762, "objective/entropy": -80.41490173, "objective/non_score_reward": -2.20632029, "objective/rlhf_reward": 11.42649174, "objective/scores": 13.625, "policy/approxkl_avg": 0.00091768, "policy/clipfrac_avg": 0.00506936, "loss/policy_avg": -0.05700582, "loss/value_avg": 0.62412775, "val/clipfrac_avg": 0.18845901, "policy/entropy_avg": 0.17818451, "val/ratio": 0.98758447, "val/ratio_var": 7.174e-05, "val/num_eos_tokens": 51, "lr": 5.94e-06, "episode": 9280, "epoch": 0.46868687, "global_step/max_steps": "580/1238", "percentage": "46.85%", "elapsed_time": "7h 5m 21s", "remaining_time": "8h 2m 33s"} +{"eps": 0, "objective/kl": 37.18535614, "objective/entropy": -76.9342804, "objective/non_score_reward": -1.85926795, "objective/rlhf_reward": 12.87510681, "objective/scores": 14.75, "policy/approxkl_avg": 0.00097987, "policy/clipfrac_avg": 0.00777215, "loss/policy_avg": -0.0618856, "loss/value_avg": 0.3465094, "val/clipfrac_avg": 0.06157155, "policy/entropy_avg": 0.17570305, "val/ratio": 0.98813689, "val/ratio_var": 4.801e-05, "val/num_eos_tokens": 158, "lr": 5.93e-06, "episode": 9296, "epoch": 0.46949495, "global_step/max_steps": "581/1238", "percentage": "46.93%", "elapsed_time": "7h 6m 22s", "remaining_time": "8h 2m 9s"} +{"eps": 0, "objective/kl": 39.92691803, "objective/entropy": -64.74710846, "objective/non_score_reward": -1.99634612, "objective/rlhf_reward": 12.65990353, "objective/scores": 14.625, "policy/approxkl_avg": 0.00091706, "policy/clipfrac_avg": 0.00733519, "loss/policy_avg": 0.13406801, "loss/value_avg": 0.67180246, "val/clipfrac_avg": 0.10079885, "policy/entropy_avg": 0.1812706, "val/ratio": 0.98818803, "val/ratio_var": 5.956e-05, "val/num_eos_tokens": 383, "lr": 5.92e-06, "episode": 9312, "epoch": 0.47030303, "global_step/max_steps": "582/1238", "percentage": "47.01%", "elapsed_time": "7h 6m 56s", "remaining_time": "8h 1m 13s"} +{"eps": 0, "objective/kl": 32.04585648, "objective/entropy": -120.16492462, "objective/non_score_reward": -1.60229278, "objective/rlhf_reward": 13.27270699, "objective/scores": 14.875, "policy/approxkl_avg": 0.00092712, "policy/clipfrac_avg": 0.01571074, "loss/policy_avg": 0.35301957, "loss/value_avg": 0.40710646, "val/clipfrac_avg": 0.0412995, "policy/entropy_avg": 0.14319611, "val/ratio": 0.98933136, "val/ratio_var": 5.184e-05, "val/num_eos_tokens": 319, "lr": 5.9e-06, "episode": 9328, "epoch": 0.47111111, "global_step/max_steps": "583/1238", "percentage": "47.09%", "elapsed_time": "7h 7m 40s", "remaining_time": "8h 0m 30s"} +{"eps": 0, "objective/kl": 37.35072327, "objective/entropy": -21.60655975, "objective/non_score_reward": -1.86753631, "objective/rlhf_reward": 13.48402596, "objective/scores": 15.375, "policy/approxkl_avg": 0.00110012, "policy/clipfrac_avg": 0.01022223, "loss/policy_avg": -0.03923231, "loss/value_avg": 0.56453669, "val/clipfrac_avg": 0.09733504, "policy/entropy_avg": 0.17325497, "val/ratio": 0.98910105, "val/ratio_var": 5.214e-05, "val/num_eos_tokens": 8, "lr": 5.89e-06, "episode": 9344, "epoch": 0.47191919, "global_step/max_steps": "584/1238", "percentage": "47.17%", "elapsed_time": "7h 8m 20s", "remaining_time": "7h 59m 40s"} +{"eps": 0, "objective/kl": 43.35786438, "objective/entropy": -169.64785767, "objective/non_score_reward": -2.16789317, "objective/rlhf_reward": 12.07429504, "objective/scores": 14.25, "policy/approxkl_avg": 0.00073536, "policy/clipfrac_avg": 0.00401792, "loss/policy_avg": -0.27854422, "loss/value_avg": 0.58312458, "val/clipfrac_avg": 0.21491241, "policy/entropy_avg": 0.13879728, "val/ratio": 0.99077678, "val/ratio_var": 4.721e-05, "val/num_eos_tokens": 92, "lr": 5.88e-06, "episode": 9360, "epoch": 0.47272727, "global_step/max_steps": "585/1238", "percentage": "47.25%", "elapsed_time": "7h 10m 4s", "remaining_time": "8h 0m 4s"} +{"eps": 0, "objective/kl": 49.14929199, "objective/entropy": -151.753479, "objective/non_score_reward": -2.45746469, "objective/rlhf_reward": 11.22222233, "objective/scores": 13.6875, "policy/approxkl_avg": 0.00090737, "policy/clipfrac_avg": 0.00646247, "loss/policy_avg": 0.05107239, "loss/value_avg": 0.36724406, "val/clipfrac_avg": 0.09604116, "policy/entropy_avg": 0.1743145, "val/ratio": 0.98881233, "val/ratio_var": 3.679e-05, "val/num_eos_tokens": 162, "lr": 5.86e-06, "episode": 9376, "epoch": 0.47353535, "global_step/max_steps": "586/1238", "percentage": "47.33%", "elapsed_time": "7h 11m 50s", "remaining_time": "8h 0m 28s"} +{"eps": 0, "objective/kl": 36.87356567, "objective/entropy": -66.69998932, "objective/non_score_reward": -1.84367824, "objective/rlhf_reward": 12.39850903, "objective/scores": 14.25, "policy/approxkl_avg": 0.0008563, "policy/clipfrac_avg": 0.0148137, "loss/policy_avg": -0.07891326, "loss/value_avg": 0.20245111, "val/clipfrac_avg": 0.04891028, "policy/entropy_avg": 0.18907166, "val/ratio": 0.9900521, "val/ratio_var": 4.238e-05, "val/num_eos_tokens": 361, "lr": 5.85e-06, "episode": 9392, "epoch": 0.47434343, "global_step/max_steps": "587/1238", "percentage": "47.42%", "elapsed_time": "7h 12m 20s", "remaining_time": "7h 59m 28s"} +{"eps": 0, "objective/kl": 37.9536438, "objective/entropy": -44.46873093, "objective/non_score_reward": -1.89768219, "objective/rlhf_reward": 13.44606781, "objective/scores": 15.375, "policy/approxkl_avg": 0.00103756, "policy/clipfrac_avg": 0.00637188, "loss/policy_avg": 0.05903597, "loss/value_avg": 0.44623643, "val/clipfrac_avg": 0.14833212, "policy/entropy_avg": 0.19812298, "val/ratio": 0.98679316, "val/ratio_var": 4.73e-05, "val/num_eos_tokens": 409, "lr": 5.84e-06, "episode": 9408, "epoch": 0.47515152, "global_step/max_steps": "588/1238", "percentage": "47.50%", "elapsed_time": "7h 12m 53s", "remaining_time": "7h 58m 32s"} +{"eps": 0, "objective/kl": 38.77004242, "objective/entropy": -42.06518555, "objective/non_score_reward": -1.93850219, "objective/rlhf_reward": 12.96774864, "objective/scores": 14.875, "policy/approxkl_avg": 0.00114387, "policy/clipfrac_avg": 0.01034562, "loss/policy_avg": 0.10919663, "loss/value_avg": 0.36613423, "val/clipfrac_avg": 0.05800151, "policy/entropy_avg": 0.20827293, "val/ratio": 0.98612463, "val/ratio_var": 5.323e-05, "val/num_eos_tokens": 131, "lr": 5.82e-06, "episode": 9424, "epoch": 0.4759596, "global_step/max_steps": "589/1238", "percentage": "47.58%", "elapsed_time": "7h 13m 26s", "remaining_time": "7h 57m 35s"} +{"eps": 0, "objective/kl": 43.38071442, "objective/entropy": -48.18125153, "objective/non_score_reward": -2.16903591, "objective/rlhf_reward": 15.26065063, "objective/scores": 17.375, "policy/approxkl_avg": 0.00095711, "policy/clipfrac_avg": 0.00715309, "loss/policy_avg": -0.10685921, "loss/value_avg": 0.60443217, "val/clipfrac_avg": 0.18321609, "policy/entropy_avg": 0.18917751, "val/ratio": 0.98763037, "val/ratio_var": 7.049e-05, "val/num_eos_tokens": 322, "lr": 5.81e-06, "episode": 9440, "epoch": 0.47676768, "global_step/max_steps": "590/1238", "percentage": "47.66%", "elapsed_time": "7h 14m 3s", "remaining_time": "7h 56m 43s"} +{"eps": 0, "objective/kl": 38.93789673, "objective/entropy": -93.83703613, "objective/non_score_reward": -1.94689488, "objective/rlhf_reward": 13.22498035, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00091194, "policy/clipfrac_avg": 0.01098588, "loss/policy_avg": -0.14776897, "loss/value_avg": 0.4778249, "val/clipfrac_avg": 0.07069977, "policy/entropy_avg": 0.17043495, "val/ratio": 0.9888739, "val/ratio_var": 5.067e-05, "val/num_eos_tokens": 209, "lr": 5.8e-06, "episode": 9456, "epoch": 0.47757576, "global_step/max_steps": "591/1238", "percentage": "47.74%", "elapsed_time": "7h 14m 49s", "remaining_time": "7h 56m 1s"} +{"eps": 0, "objective/kl": 35.89628601, "objective/entropy": -33.68360901, "objective/non_score_reward": -1.79481411, "objective/rlhf_reward": 12.59581089, "objective/scores": 14.375, "policy/approxkl_avg": 0.00089121, "policy/clipfrac_avg": 0.01152383, "loss/policy_avg": 0.14408515, "loss/value_avg": 0.55679381, "val/clipfrac_avg": 0.15322092, "policy/entropy_avg": 0.13869476, "val/ratio": 0.9900077, "val/ratio_var": 2.749e-05, "val/num_eos_tokens": 79, "lr": 5.78e-06, "episode": 9472, "epoch": 0.47838384, "global_step/max_steps": "592/1238", "percentage": "47.82%", "elapsed_time": "7h 15m 17s", "remaining_time": "7h 54m 59s"} +{"eps": 0, "objective/kl": 34.06424332, "objective/entropy": -79.1028595, "objective/non_score_reward": -1.70321226, "objective/rlhf_reward": 12.25772476, "objective/scores": 13.9375, "policy/approxkl_avg": 0.00093094, "policy/clipfrac_avg": 0.0094521, "loss/policy_avg": 0.06612088, "loss/value_avg": 0.58217221, "val/clipfrac_avg": 0.11116342, "policy/entropy_avg": 0.15736389, "val/ratio": 0.98953438, "val/ratio_var": 4.008e-05, "val/num_eos_tokens": 63, "lr": 5.77e-06, "episode": 9488, "epoch": 0.47919192, "global_step/max_steps": "593/1238", "percentage": "47.90%", "elapsed_time": "7h 15m 55s", "remaining_time": "7h 54m 9s"} +{"eps": 0, "objective/kl": 44.31615067, "objective/entropy": -173.360672, "objective/non_score_reward": -2.21580768, "objective/rlhf_reward": 11.31544209, "objective/scores": 13.5, "policy/approxkl_avg": 0.00080138, "policy/clipfrac_avg": 0.00928047, "loss/policy_avg": -0.09432862, "loss/value_avg": 0.27976608, "val/clipfrac_avg": 0.04634128, "policy/entropy_avg": 0.15043259, "val/ratio": 0.99043161, "val/ratio_var": 5.513e-05, "val/num_eos_tokens": 439, "lr": 5.76e-06, "episode": 9504, "epoch": 0.48, "global_step/max_steps": "594/1238", "percentage": "47.98%", "elapsed_time": "7h 17m 40s", "remaining_time": "7h 54m 31s"} +{"eps": 0, "objective/kl": 39.01966095, "objective/entropy": -66.67430878, "objective/non_score_reward": -1.95098329, "objective/rlhf_reward": 14.34589195, "objective/scores": 16.25, "policy/approxkl_avg": 0.00099614, "policy/clipfrac_avg": 0.0066235, "loss/policy_avg": -0.17459822, "loss/value_avg": 0.56894398, "val/clipfrac_avg": 0.09633146, "policy/entropy_avg": 0.17397308, "val/ratio": 0.98878598, "val/ratio_var": 4.247e-05, "val/num_eos_tokens": 305, "lr": 5.75e-06, "episode": 9520, "epoch": 0.48080808, "global_step/max_steps": "595/1238", "percentage": "48.06%", "elapsed_time": "7h 18m 20s", "remaining_time": "7h 53m 41s"} +{"eps": 0, "objective/kl": 36.13384247, "objective/entropy": -38.30708313, "objective/non_score_reward": -1.80669224, "objective/rlhf_reward": 14.46674538, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009535, "policy/clipfrac_avg": 0.00545323, "loss/policy_avg": -0.06750008, "loss/value_avg": 0.25791821, "val/clipfrac_avg": 0.08646031, "policy/entropy_avg": 0.17740631, "val/ratio": 0.98766887, "val/ratio_var": 3.323e-05, "val/num_eos_tokens": 232, "lr": 5.73e-06, "episode": 9536, "epoch": 0.48161616, "global_step/max_steps": "596/1238", "percentage": "48.14%", "elapsed_time": "7h 18m 52s", "remaining_time": "7h 52m 45s"} +{"eps": 0, "objective/kl": 25.86457443, "objective/entropy": -108.50715637, "objective/non_score_reward": -1.29322875, "objective/rlhf_reward": 12.94114685, "objective/scores": 14.25, "policy/approxkl_avg": 0.00072678, "policy/clipfrac_avg": 0.02497327, "loss/policy_avg": 0.08241126, "loss/value_avg": 1.29659903, "val/clipfrac_avg": 0.03562643, "policy/entropy_avg": 0.12932777, "val/ratio": 0.99111807, "val/ratio_var": 5.247e-05, "val/num_eos_tokens": 90, "lr": 5.72e-06, "episode": 9552, "epoch": 0.48242424, "global_step/max_steps": "597/1238", "percentage": "48.22%", "elapsed_time": "7h 19m 36s", "remaining_time": "7h 52m 0s"} +{"eps": 0, "objective/kl": 42.28157806, "objective/entropy": -73.05348969, "objective/non_score_reward": -2.114079, "objective/rlhf_reward": 14.26092148, "objective/scores": 16.375, "policy/approxkl_avg": 0.00109071, "policy/clipfrac_avg": 0.00787961, "loss/policy_avg": -0.03618451, "loss/value_avg": 0.27752417, "val/clipfrac_avg": 0.05385239, "policy/entropy_avg": 0.18637657, "val/ratio": 0.98733985, "val/ratio_var": 5.952e-05, "val/num_eos_tokens": 278, "lr": 5.71e-06, "episode": 9568, "epoch": 0.48323232, "global_step/max_steps": "598/1238", "percentage": "48.30%", "elapsed_time": "7h 20m 14s", "remaining_time": "7h 51m 9s"} +{"eps": 0, "objective/kl": 36.10977173, "objective/entropy": -93.86756897, "objective/non_score_reward": -1.80548847, "objective/rlhf_reward": 12.61638641, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00106267, "policy/clipfrac_avg": 0.00648365, "loss/policy_avg": -0.0933134, "loss/value_avg": 0.43138564, "val/clipfrac_avg": 0.05310796, "policy/entropy_avg": 0.15071487, "val/ratio": 0.98954988, "val/ratio_var": 5.325e-05, "val/num_eos_tokens": 201, "lr": 5.69e-06, "episode": 9584, "epoch": 0.4840404, "global_step/max_steps": "599/1238", "percentage": "48.38%", "elapsed_time": "7h 21m 25s", "remaining_time": "7h 50m 53s"} +{"eps": 0, "objective/kl": 45.25263977, "objective/entropy": -186.87242126, "objective/non_score_reward": -2.26263189, "objective/rlhf_reward": 13.53424263, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00074875, "policy/clipfrac_avg": 0.00282776, "loss/policy_avg": -0.15645877, "loss/value_avg": 0.32119846, "val/clipfrac_avg": 0.08722794, "policy/entropy_avg": 0.15339661, "val/ratio": 0.98996055, "val/ratio_var": 3.863e-05, "val/num_eos_tokens": 503, "lr": 5.68e-06, "episode": 9600, "epoch": 0.48484848, "global_step/max_steps": "600/1238", "percentage": "48.47%", "elapsed_time": "7h 23m 5s", "remaining_time": "7h 51m 9s"} +{"eps": 0, "objective/kl": 41.66707611, "objective/entropy": -43.57266235, "objective/non_score_reward": -2.083354, "objective/rlhf_reward": 12.33852005, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00127294, "policy/clipfrac_avg": 0.01519966, "loss/policy_avg": 0.26468652, "loss/value_avg": 0.95883608, "val/clipfrac_avg": 0.20558898, "policy/entropy_avg": 0.18356419, "val/ratio": 0.98918521, "val/ratio_var": 8.858e-05, "val/num_eos_tokens": 18, "lr": 5.67e-06, "episode": 9616, "epoch": 0.48565657, "global_step/max_steps": "601/1238", "percentage": "48.55%", "elapsed_time": "7h 23m 57s", "remaining_time": "7h 50m 33s"} +{"eps": 0, "objective/kl": 40.64360809, "objective/entropy": -80.89665985, "objective/non_score_reward": -2.03218031, "objective/rlhf_reward": 13.13188171, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00091472, "policy/clipfrac_avg": 0.00729462, "loss/policy_avg": 0.01831547, "loss/value_avg": 0.53220034, "val/clipfrac_avg": 0.08266787, "policy/entropy_avg": 0.17550898, "val/ratio": 0.98807627, "val/ratio_var": 7.163e-05, "val/num_eos_tokens": 181, "lr": 5.65e-06, "episode": 9632, "epoch": 0.48646465, "global_step/max_steps": "602/1238", "percentage": "48.63%", "elapsed_time": "7h 24m 42s", "remaining_time": "7h 49m 49s"} +{"eps": 0, "objective/kl": 37.01217651, "objective/entropy": -56.91044617, "objective/non_score_reward": -1.85060883, "objective/rlhf_reward": 13.82126617, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00084625, "policy/clipfrac_avg": 0.00393457, "loss/policy_avg": -0.22014892, "loss/value_avg": 0.27795735, "val/clipfrac_avg": 0.07405911, "policy/entropy_avg": 0.15899277, "val/ratio": 0.9893809, "val/ratio_var": 5.063e-05, "val/num_eos_tokens": 119, "lr": 5.64e-06, "episode": 9648, "epoch": 0.48727273, "global_step/max_steps": "603/1238", "percentage": "48.71%", "elapsed_time": "7h 25m 14s", "remaining_time": "7h 48m 52s"} +{"eps": 0, "objective/kl": 45.01192474, "objective/entropy": 0.57705021, "objective/non_score_reward": -2.25059605, "objective/rlhf_reward": 14.390028, "objective/scores": 16.625, "policy/approxkl_avg": 0.00114453, "policy/clipfrac_avg": 0.00484386, "loss/policy_avg": -0.1312999, "loss/value_avg": 0.29017082, "val/clipfrac_avg": 0.05336828, "policy/entropy_avg": 0.21940231, "val/ratio": 0.98539054, "val/ratio_var": 4.156e-05, "val/num_eos_tokens": 82, "lr": 5.63e-06, "episode": 9664, "epoch": 0.48808081, "global_step/max_steps": "604/1238", "percentage": "48.79%", "elapsed_time": "7h 25m 55s", "remaining_time": "7h 48m 4s"} +{"eps": 0, "objective/kl": 29.19556999, "objective/entropy": -104.7314682, "objective/non_score_reward": -1.45977855, "objective/rlhf_reward": 13.54022217, "objective/scores": 15.0, "policy/approxkl_avg": 0.00074931, "policy/clipfrac_avg": 0.00599675, "loss/policy_avg": -0.15523458, "loss/value_avg": 1.38002372, "val/clipfrac_avg": 0.11536569, "policy/entropy_avg": 0.14516449, "val/ratio": 0.99022245, "val/ratio_var": 5.349e-05, "val/num_eos_tokens": 465, "lr": 5.61e-06, "episode": 9680, "epoch": 0.48888889, "global_step/max_steps": "605/1238", "percentage": "48.87%", "elapsed_time": "7h 26m 32s", "remaining_time": "7h 47m 12s"} +{"eps": 0, "objective/kl": 39.36458969, "objective/entropy": -157.14692688, "objective/non_score_reward": -1.96822953, "objective/rlhf_reward": 12.36380196, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00085823, "policy/clipfrac_avg": 0.010594, "loss/policy_avg": -0.27331144, "loss/value_avg": 1.16130531, "val/clipfrac_avg": 0.06608556, "policy/entropy_avg": 0.14953947, "val/ratio": 0.98991334, "val/ratio_var": 4.76e-05, "val/num_eos_tokens": 412, "lr": 5.6e-06, "episode": 9696, "epoch": 0.48969697, "global_step/max_steps": "606/1238", "percentage": "48.95%", "elapsed_time": "7h 27m 26s", "remaining_time": "7h 46m 37s"} +{"eps": 0, "objective/kl": 33.51747131, "objective/entropy": -105.65791321, "objective/non_score_reward": -1.67587364, "objective/rlhf_reward": 13.28115749, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00088414, "policy/clipfrac_avg": 0.00696577, "loss/policy_avg": -0.198709, "loss/value_avg": 0.37283677, "val/clipfrac_avg": 0.0555139, "policy/entropy_avg": 0.15077972, "val/ratio": 0.98943675, "val/ratio_var": 7.082e-05, "val/num_eos_tokens": 146, "lr": 5.59e-06, "episode": 9712, "epoch": 0.49050505, "global_step/max_steps": "607/1238", "percentage": "49.03%", "elapsed_time": "7h 27m 55s", "remaining_time": "7h 45m 37s"} +{"eps": 0, "objective/kl": 36.81121063, "objective/entropy": -30.99787521, "objective/non_score_reward": -1.84056056, "objective/rlhf_reward": 12.97193909, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00100208, "policy/clipfrac_avg": 0.0098721, "loss/policy_avg": 0.12182619, "loss/value_avg": 0.64987779, "val/clipfrac_avg": 0.0928297, "policy/entropy_avg": 0.161026, "val/ratio": 0.98794448, "val/ratio_var": 3.414e-05, "val/num_eos_tokens": 288, "lr": 5.57e-06, "episode": 9728, "epoch": 0.49131313, "global_step/max_steps": "608/1238", "percentage": "49.11%", "elapsed_time": "7h 28m 23s", "remaining_time": "7h 44m 37s"} +{"eps": 0, "objective/kl": 36.83974838, "objective/entropy": -79.70800781, "objective/non_score_reward": -1.84198761, "objective/rlhf_reward": 11.47832489, "objective/scores": 13.3125, "policy/approxkl_avg": 0.00088992, "policy/clipfrac_avg": 0.01262018, "loss/policy_avg": -0.11768663, "loss/value_avg": 0.79921657, "val/clipfrac_avg": 0.05175953, "policy/entropy_avg": 0.15257454, "val/ratio": 0.98973745, "val/ratio_var": 4.772e-05, "val/num_eos_tokens": 13, "lr": 5.56e-06, "episode": 9744, "epoch": 0.49212121, "global_step/max_steps": "609/1238", "percentage": "49.19%", "elapsed_time": "7h 28m 59s", "remaining_time": "7h 43m 44s"} +{"eps": 0, "objective/kl": 33.13558578, "objective/entropy": -32.14586639, "objective/non_score_reward": -1.65677941, "objective/rlhf_reward": 12.78462601, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00126701, "policy/clipfrac_avg": 0.00890309, "loss/policy_avg": -0.24540734, "loss/value_avg": 1.01817656, "val/clipfrac_avg": 0.12171463, "policy/entropy_avg": 0.18150234, "val/ratio": 0.9867506, "val/ratio_var": 5.663e-05, "val/num_eos_tokens": 215, "lr": 5.55e-06, "episode": 9760, "epoch": 0.49292929, "global_step/max_steps": "610/1238", "percentage": "49.27%", "elapsed_time": "7h 29m 30s", "remaining_time": "7h 42m 46s"} +{"eps": 0, "objective/kl": 32.43745804, "objective/entropy": -21.85140419, "objective/non_score_reward": -1.62187302, "objective/rlhf_reward": 15.72969055, "objective/scores": 17.375, "policy/approxkl_avg": 0.00159641, "policy/clipfrac_avg": 0.01003016, "loss/policy_avg": -0.05234598, "loss/value_avg": 0.95063722, "val/clipfrac_avg": 0.19708402, "policy/entropy_avg": 0.19261169, "val/ratio": 0.98594725, "val/ratio_var": 9.207e-05, "val/num_eos_tokens": 8, "lr": 5.53e-06, "episode": 9776, "epoch": 0.49373737, "global_step/max_steps": "611/1238", "percentage": "49.35%", "elapsed_time": "7h 30m 3s", "remaining_time": "7h 41m 50s"} +{"eps": 0, "objective/kl": 35.33883286, "objective/entropy": -77.9874115, "objective/non_score_reward": -1.76694167, "objective/rlhf_reward": 14.93618393, "objective/scores": 16.75, "policy/approxkl_avg": 0.00079285, "policy/clipfrac_avg": 0.0295839, "loss/policy_avg": -0.00216074, "loss/value_avg": 0.45648712, "val/clipfrac_avg": 0.07284848, "policy/entropy_avg": 0.14488077, "val/ratio": 0.99064338, "val/ratio_var": 5.276e-05, "val/num_eos_tokens": 735, "lr": 5.52e-06, "episode": 9792, "epoch": 0.49454545, "global_step/max_steps": "612/1238", "percentage": "49.43%", "elapsed_time": "7h 30m 47s", "remaining_time": "7h 41m 6s"} +{"eps": 0, "objective/kl": 38.19306183, "objective/entropy": -163.56391907, "objective/non_score_reward": -1.90965295, "objective/rlhf_reward": 11.53565979, "objective/scores": 13.4375, "policy/approxkl_avg": 0.0008345, "policy/clipfrac_avg": 0.00699893, "loss/policy_avg": -0.08518673, "loss/value_avg": 0.62959635, "val/clipfrac_avg": 0.04129464, "policy/entropy_avg": 0.21862793, "val/ratio": 0.98951685, "val/ratio_var": 4.985e-05, "val/num_eos_tokens": 393, "lr": 5.51e-06, "episode": 9808, "epoch": 0.49535354, "global_step/max_steps": "613/1238", "percentage": "49.52%", "elapsed_time": "7h 32m 28s", "remaining_time": "7h 41m 19s"} +{"eps": 0, "objective/kl": 36.03221893, "objective/entropy": -78.88749695, "objective/non_score_reward": -1.80161095, "objective/rlhf_reward": 13.74526405, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00105432, "policy/clipfrac_avg": 0.01225765, "loss/policy_avg": -0.09563255, "loss/value_avg": 0.33951002, "val/clipfrac_avg": 0.03431977, "policy/entropy_avg": 0.17931032, "val/ratio": 0.98784804, "val/ratio_var": 7.281e-05, "val/num_eos_tokens": 348, "lr": 5.49e-06, "episode": 9824, "epoch": 0.49616162, "global_step/max_steps": "614/1238", "percentage": "49.60%", "elapsed_time": "7h 33m 24s", "remaining_time": "7h 40m 47s"} +{"eps": 0, "objective/kl": 29.16685486, "objective/entropy": -71.89592743, "objective/non_score_reward": -1.45834279, "objective/rlhf_reward": 11.68228245, "objective/scores": 13.125, "policy/approxkl_avg": 0.00100098, "policy/clipfrac_avg": 0.00919897, "loss/policy_avg": -0.01123824, "loss/value_avg": 1.18441868, "val/clipfrac_avg": 0.15483876, "policy/entropy_avg": 0.18142796, "val/ratio": 0.99105668, "val/ratio_var": 0.00011891, "val/num_eos_tokens": 405, "lr": 5.48e-06, "episode": 9840, "epoch": 0.4969697, "global_step/max_steps": "615/1238", "percentage": "49.68%", "elapsed_time": "7h 33m 51s", "remaining_time": "7h 39m 45s"} +{"eps": 0, "objective/kl": 39.84773636, "objective/entropy": -75.90268707, "objective/non_score_reward": -1.99238682, "objective/rlhf_reward": 10.14823818, "objective/scores": 12.125, "policy/approxkl_avg": 0.00125643, "policy/clipfrac_avg": 0.00964525, "loss/policy_avg": 0.07725506, "loss/value_avg": 1.20750749, "val/clipfrac_avg": 0.09601637, "policy/entropy_avg": 0.1919899, "val/ratio": 0.98737526, "val/ratio_var": 8.283e-05, "val/num_eos_tokens": 432, "lr": 5.47e-06, "episode": 9856, "epoch": 0.49777778, "global_step/max_steps": "616/1238", "percentage": "49.76%", "elapsed_time": "7h 34m 28s", "remaining_time": "7h 38m 54s"} +{"eps": 0, "objective/kl": 39.15164948, "objective/entropy": -117.68332672, "objective/non_score_reward": -1.95758259, "objective/rlhf_reward": 13.11273003, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00080852, "policy/clipfrac_avg": 0.01115941, "loss/policy_avg": -0.12253916, "loss/value_avg": 0.34518784, "val/clipfrac_avg": 0.08139624, "policy/entropy_avg": 0.16089964, "val/ratio": 0.98969948, "val/ratio_var": 3.962e-05, "val/num_eos_tokens": 740, "lr": 5.45e-06, "episode": 9872, "epoch": 0.49858586, "global_step/max_steps": "617/1238", "percentage": "49.84%", "elapsed_time": "7h 35m 49s", "remaining_time": "7h 38m 47s"} +{"eps": 0, "objective/kl": 27.95919418, "objective/entropy": -76.56217194, "objective/non_score_reward": -1.39795971, "objective/rlhf_reward": 15.20360374, "objective/scores": 16.625, "policy/approxkl_avg": 0.00086536, "policy/clipfrac_avg": 0.00950298, "loss/policy_avg": 0.1359424, "loss/value_avg": 0.69781101, "val/clipfrac_avg": 0.03875832, "policy/entropy_avg": 0.16449261, "val/ratio": 0.98847228, "val/ratio_var": 7.257e-05, "val/num_eos_tokens": 5, "lr": 5.44e-06, "episode": 9888, "epoch": 0.49939394, "global_step/max_steps": "618/1238", "percentage": "49.92%", "elapsed_time": "7h 36m 27s", "remaining_time": "7h 37m 56s"} +{"eps": 0, "objective/kl": 35.96702194, "objective/entropy": -51.29543686, "objective/non_score_reward": -1.79835105, "objective/rlhf_reward": 13.36571121, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00109041, "policy/clipfrac_avg": 0.00761054, "loss/policy_avg": 0.03163297, "loss/value_avg": 0.59133291, "val/clipfrac_avg": 0.18358186, "policy/entropy_avg": 0.18178225, "val/ratio": 0.98818839, "val/ratio_var": 4.506e-05, "val/num_eos_tokens": 139, "lr": 5.43e-06, "episode": 9904, "epoch": 0.50020202, "global_step/max_steps": "619/1238", "percentage": "50.00%", "elapsed_time": "7h 37m 5s", "remaining_time": "7h 37m 5s"} +{"eps": 0, "objective/kl": 41.84601593, "objective/entropy": -40.40581131, "objective/non_score_reward": -2.09230089, "objective/rlhf_reward": 14.43113708, "objective/scores": 16.5, "policy/approxkl_avg": 0.00099335, "policy/clipfrac_avg": 0.00414526, "loss/policy_avg": -0.01861837, "loss/value_avg": 0.14973307, "val/clipfrac_avg": 0.01799292, "policy/entropy_avg": 0.19639587, "val/ratio": 0.98763418, "val/ratio_var": 2.161e-05, "val/num_eos_tokens": 503, "lr": 5.41e-06, "episode": 9920, "epoch": 0.5010101, "global_step/max_steps": "620/1238", "percentage": "50.08%", "elapsed_time": "7h 37m 49s", "remaining_time": "7h 36m 20s"} +{"eps": 0, "objective/kl": 34.00743103, "objective/entropy": -250.99803162, "objective/non_score_reward": -1.70037162, "objective/rlhf_reward": 13.35431671, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00081347, "policy/clipfrac_avg": 0.01992632, "loss/policy_avg": 0.07495287, "loss/value_avg": 0.62118924, "val/clipfrac_avg": 0.060361, "policy/entropy_avg": 0.15630341, "val/ratio": 0.99133265, "val/ratio_var": 4.28e-05, "val/num_eos_tokens": 21, "lr": 5.4e-06, "episode": 9936, "epoch": 0.50181818, "global_step/max_steps": "621/1238", "percentage": "50.16%", "elapsed_time": "7h 38m 54s", "remaining_time": "7h 35m 57s"} +{"eps": 0, "objective/kl": 22.70136642, "objective/entropy": -123.36508942, "objective/non_score_reward": -1.1350683, "objective/rlhf_reward": 13.37274456, "objective/scores": 14.5, "policy/approxkl_avg": 0.00099353, "policy/clipfrac_avg": 0.00878138, "loss/policy_avg": 0.3656258, "loss/value_avg": 1.01105201, "val/clipfrac_avg": 0.1413438, "policy/entropy_avg": 0.11741734, "val/ratio": 0.99146402, "val/ratio_var": 6.539e-05, "val/num_eos_tokens": 221, "lr": 5.39e-06, "episode": 9952, "epoch": 0.50262626, "global_step/max_steps": "622/1238", "percentage": "50.24%", "elapsed_time": "7h 39m 32s", "remaining_time": "7h 35m 6s"} +{"eps": 0, "objective/kl": 36.58520126, "objective/entropy": -120.33831787, "objective/non_score_reward": -1.82926011, "objective/rlhf_reward": 12.94417763, "objective/scores": 14.75, "policy/approxkl_avg": 0.00159831, "policy/clipfrac_avg": 0.02764862, "loss/policy_avg": 0.21662173, "loss/value_avg": 0.46216172, "val/clipfrac_avg": 0.17286101, "policy/entropy_avg": 0.19037724, "val/ratio": 0.98668712, "val/ratio_var": 0.00018466, "val/num_eos_tokens": 333, "lr": 5.37e-06, "episode": 9968, "epoch": 0.50343434, "global_step/max_steps": "623/1238", "percentage": "50.32%", "elapsed_time": "7h 40m 11s", "remaining_time": "7h 34m 17s"} +{"eps": 0, "objective/kl": 34.64921951, "objective/entropy": -107.19503784, "objective/non_score_reward": -1.73246074, "objective/rlhf_reward": 11.34566402, "objective/scores": 13.0625, "policy/approxkl_avg": 0.00096915, "policy/clipfrac_avg": 0.00613647, "loss/policy_avg": 0.03029388, "loss/value_avg": 0.54228938, "val/clipfrac_avg": 0.06680267, "policy/entropy_avg": 0.19006157, "val/ratio": 0.98823321, "val/ratio_var": 7.252e-05, "val/num_eos_tokens": 650, "lr": 5.36e-06, "episode": 9984, "epoch": 0.50424242, "global_step/max_steps": "624/1238", "percentage": "50.40%", "elapsed_time": "7h 40m 51s", "remaining_time": "7h 33m 28s"} +{"eps": 0, "objective/kl": 37.40558624, "objective/entropy": -101.55954742, "objective/non_score_reward": -1.87027943, "objective/rlhf_reward": 13.19222069, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00089682, "policy/clipfrac_avg": 0.01471101, "loss/policy_avg": 0.24651952, "loss/value_avg": 0.68727314, "val/clipfrac_avg": 0.09929686, "policy/entropy_avg": 0.14348412, "val/ratio": 0.98885059, "val/ratio_var": 4.519e-05, "val/num_eos_tokens": 428, "lr": 5.35e-06, "episode": 10000, "epoch": 0.50505051, "global_step/max_steps": "625/1238", "percentage": "50.48%", "elapsed_time": "7h 41m 20s", "remaining_time": "7h 32m 29s"} +{"eps": 0, "objective/kl": 39.13426971, "objective/entropy": -147.00888062, "objective/non_score_reward": -1.9567138, "objective/rlhf_reward": 14.19172382, "objective/scores": 16.125, "policy/approxkl_avg": 0.00085014, "policy/clipfrac_avg": 0.0089168, "loss/policy_avg": -0.09769875, "loss/value_avg": 0.26997975, "val/clipfrac_avg": 0.07781403, "policy/entropy_avg": 0.19576263, "val/ratio": 0.98836398, "val/ratio_var": 7.028e-05, "val/num_eos_tokens": 156, "lr": 5.33e-06, "episode": 10016, "epoch": 0.50585859, "global_step/max_steps": "626/1238", "percentage": "50.57%", "elapsed_time": "7h 42m 0s", "remaining_time": "7h 31m 40s"} +{"eps": 0, "objective/kl": 31.40962219, "objective/entropy": -60.07448196, "objective/non_score_reward": -1.57048106, "objective/rlhf_reward": 14.78889465, "objective/scores": 16.375, "policy/approxkl_avg": 0.00085948, "policy/clipfrac_avg": 0.00705072, "loss/policy_avg": -0.15752657, "loss/value_avg": 1.41418457, "val/clipfrac_avg": 0.1173691, "policy/entropy_avg": 0.17131233, "val/ratio": 0.98841166, "val/ratio_var": 6.115e-05, "val/num_eos_tokens": 235, "lr": 5.32e-06, "episode": 10032, "epoch": 0.50666667, "global_step/max_steps": "627/1238", "percentage": "50.65%", "elapsed_time": "7h 42m 22s", "remaining_time": "7h 30m 34s"} +{"eps": 0, "objective/kl": 32.27813721, "objective/entropy": -94.50772095, "objective/non_score_reward": -1.61390686, "objective/rlhf_reward": 13.76890564, "objective/scores": 15.375, "policy/approxkl_avg": 0.00072595, "policy/clipfrac_avg": 0.0059073, "loss/policy_avg": -0.3966859, "loss/value_avg": 0.72630084, "val/clipfrac_avg": 0.24970403, "policy/entropy_avg": 0.13324738, "val/ratio": 0.99171472, "val/ratio_var": 4.824e-05, "val/num_eos_tokens": 195, "lr": 5.31e-06, "episode": 10048, "epoch": 0.50747475, "global_step/max_steps": "628/1238", "percentage": "50.73%", "elapsed_time": "7h 42m 54s", "remaining_time": "7h 29m 38s"} +{"eps": 0, "objective/kl": 30.60705185, "objective/entropy": -56.47973633, "objective/non_score_reward": -1.53035259, "objective/rlhf_reward": 12.36027145, "objective/scores": 13.875, "policy/approxkl_avg": 0.00109722, "policy/clipfrac_avg": 0.00503068, "loss/policy_avg": 0.10847074, "loss/value_avg": 1.04828191, "val/clipfrac_avg": 0.18077512, "policy/entropy_avg": 0.1625371, "val/ratio": 0.98928809, "val/ratio_var": 6.567e-05, "val/num_eos_tokens": 22, "lr": 5.29e-06, "episode": 10064, "epoch": 0.50828283, "global_step/max_steps": "629/1238", "percentage": "50.81%", "elapsed_time": "7h 43m 37s", "remaining_time": "7h 28m 52s"} +{"eps": 0, "objective/kl": 39.75598526, "objective/entropy": -135.9888916, "objective/non_score_reward": -1.98779941, "objective/rlhf_reward": 12.52001286, "objective/scores": 14.5, "policy/approxkl_avg": 0.00096514, "policy/clipfrac_avg": 0.00861822, "loss/policy_avg": -0.23807989, "loss/value_avg": 0.460262, "val/clipfrac_avg": 0.08070137, "policy/entropy_avg": 0.17929983, "val/ratio": 0.9886598, "val/ratio_var": 4.691e-05, "val/num_eos_tokens": 379, "lr": 5.28e-06, "episode": 10080, "epoch": 0.50909091, "global_step/max_steps": "630/1238", "percentage": "50.89%", "elapsed_time": "7h 44m 36s", "remaining_time": "7h 28m 22s"} +{"eps": 0, "objective/kl": 37.73855209, "objective/entropy": -94.94006348, "objective/non_score_reward": -1.8869276, "objective/rlhf_reward": 13.6286974, "objective/scores": 15.5, "policy/approxkl_avg": 0.00092479, "policy/clipfrac_avg": 0.00906059, "loss/policy_avg": 0.23663482, "loss/value_avg": 0.821612, "val/clipfrac_avg": 0.14852877, "policy/entropy_avg": 0.16341591, "val/ratio": 0.98868144, "val/ratio_var": 6.259e-05, "val/num_eos_tokens": 182, "lr": 5.27e-06, "episode": 10096, "epoch": 0.50989899, "global_step/max_steps": "631/1238", "percentage": "50.97%", "elapsed_time": "7h 45m 18s", "remaining_time": "7h 27m 36s"} +{"eps": 0, "objective/kl": 36.89660263, "objective/entropy": -80.35845947, "objective/non_score_reward": -1.84483027, "objective/rlhf_reward": 11.23329544, "objective/scores": 13.0625, "policy/approxkl_avg": 0.00122145, "policy/clipfrac_avg": 0.00661952, "loss/policy_avg": -0.08203353, "loss/value_avg": 0.84313524, "val/clipfrac_avg": 0.08380434, "policy/entropy_avg": 0.19206619, "val/ratio": 0.98824257, "val/ratio_var": 0.00010105, "val/num_eos_tokens": 526, "lr": 5.25e-06, "episode": 10112, "epoch": 0.51070707, "global_step/max_steps": "632/1238", "percentage": "51.05%", "elapsed_time": "7h 45m 57s", "remaining_time": "7h 26m 46s"} +{"eps": 0, "objective/kl": 31.87903595, "objective/entropy": -83.84121704, "objective/non_score_reward": -1.5939517, "objective/rlhf_reward": 11.90604782, "objective/scores": 13.5, "policy/approxkl_avg": 0.00077264, "policy/clipfrac_avg": 0.00810497, "loss/policy_avg": -0.20128125, "loss/value_avg": 0.36594099, "val/clipfrac_avg": 0.10230522, "policy/entropy_avg": 0.14804196, "val/ratio": 0.99033594, "val/ratio_var": 5.448e-05, "val/num_eos_tokens": 340, "lr": 5.24e-06, "episode": 10128, "epoch": 0.51151515, "global_step/max_steps": "633/1238", "percentage": "51.13%", "elapsed_time": "7h 46m 35s", "remaining_time": "7h 25m 57s"} +{"eps": 0, "objective/kl": 34.82407761, "objective/entropy": -79.21052551, "objective/non_score_reward": -1.74120402, "objective/rlhf_reward": 12.86817074, "objective/scores": 14.625, "policy/approxkl_avg": 0.00095237, "policy/clipfrac_avg": 0.00944553, "loss/policy_avg": 0.45129338, "loss/value_avg": 1.2565788, "val/clipfrac_avg": 0.24038407, "policy/entropy_avg": 0.18542004, "val/ratio": 0.98785728, "val/ratio_var": 5.598e-05, "val/num_eos_tokens": 511, "lr": 5.23e-06, "episode": 10144, "epoch": 0.51232323, "global_step/max_steps": "634/1238", "percentage": "51.21%", "elapsed_time": "7h 47m 16s", "remaining_time": "7h 25m 9s"} +{"eps": 0, "objective/kl": 41.36100769, "objective/entropy": -60.33174133, "objective/non_score_reward": -2.06805038, "objective/rlhf_reward": 13.40851212, "objective/scores": 15.5, "policy/approxkl_avg": 0.00126486, "policy/clipfrac_avg": 0.01303275, "loss/policy_avg": -0.22091357, "loss/value_avg": 0.45468521, "val/clipfrac_avg": 0.04903885, "policy/entropy_avg": 0.1959362, "val/ratio": 0.98762453, "val/ratio_var": 5.173e-05, "val/num_eos_tokens": 286, "lr": 5.21e-06, "episode": 10160, "epoch": 0.51313131, "global_step/max_steps": "635/1238", "percentage": "51.29%", "elapsed_time": "7h 47m 55s", "remaining_time": "7h 24m 20s"} +{"eps": 0, "objective/kl": 30.60900307, "objective/entropy": -65.56263733, "objective/non_score_reward": -1.53045011, "objective/rlhf_reward": 12.98517513, "objective/scores": 14.5, "policy/approxkl_avg": 0.00090483, "policy/clipfrac_avg": 0.00968816, "loss/policy_avg": 0.30760646, "loss/value_avg": 1.33135414, "val/clipfrac_avg": 0.08400048, "policy/entropy_avg": 0.16722918, "val/ratio": 0.98894972, "val/ratio_var": 5.892e-05, "val/num_eos_tokens": 173, "lr": 5.2e-06, "episode": 10176, "epoch": 0.51393939, "global_step/max_steps": "636/1238", "percentage": "51.37%", "elapsed_time": "7h 48m 26s", "remaining_time": "7h 23m 24s"} +{"eps": 0, "objective/kl": 46.15885544, "objective/entropy": -102.17712402, "objective/non_score_reward": -2.30794287, "objective/rlhf_reward": 11.96549416, "objective/scores": 14.25, "policy/approxkl_avg": 0.000975, "policy/clipfrac_avg": 0.00807768, "loss/policy_avg": -0.03748585, "loss/value_avg": 0.30425468, "val/clipfrac_avg": 0.05107585, "policy/entropy_avg": 0.16848755, "val/ratio": 0.98898846, "val/ratio_var": 4.224e-05, "val/num_eos_tokens": 307, "lr": 5.19e-06, "episode": 10192, "epoch": 0.51474747, "global_step/max_steps": "637/1238", "percentage": "51.45%", "elapsed_time": "7h 49m 6s", "remaining_time": "7h 22m 36s"} +{"eps": 0, "objective/kl": 37.84108734, "objective/entropy": -136.01560974, "objective/non_score_reward": -1.89205432, "objective/rlhf_reward": 12.2251339, "objective/scores": 14.125, "policy/approxkl_avg": 0.0011574, "policy/clipfrac_avg": 0.00875126, "loss/policy_avg": -0.23826319, "loss/value_avg": 0.73535371, "val/clipfrac_avg": 0.08653428, "policy/entropy_avg": 0.19327354, "val/ratio": 0.9871769, "val/ratio_var": 6.582e-05, "val/num_eos_tokens": 1837, "lr": 5.17e-06, "episode": 10208, "epoch": 0.51555556, "global_step/max_steps": "638/1238", "percentage": "51.53%", "elapsed_time": "7h 50m 48s", "remaining_time": "7h 22m 46s"} +{"eps": 0, "objective/kl": 37.96650314, "objective/entropy": -36.91769409, "objective/non_score_reward": -1.8983252, "objective/rlhf_reward": 14.39073753, "objective/scores": 16.25, "policy/approxkl_avg": 0.00098069, "policy/clipfrac_avg": 0.00726839, "loss/policy_avg": 0.01579358, "loss/value_avg": 0.35553756, "val/clipfrac_avg": 0.04352999, "policy/entropy_avg": 0.17595863, "val/ratio": 0.98834074, "val/ratio_var": 4.28e-05, "val/num_eos_tokens": 116, "lr": 5.16e-06, "episode": 10224, "epoch": 0.51636364, "global_step/max_steps": "639/1238", "percentage": "51.62%", "elapsed_time": "7h 51m 19s", "remaining_time": "7h 21m 49s"} +{"eps": 0, "objective/kl": 33.41941071, "objective/entropy": -150.77487183, "objective/non_score_reward": -1.67097056, "objective/rlhf_reward": 12.96965408, "objective/scores": 14.625, "policy/approxkl_avg": 0.00076141, "policy/clipfrac_avg": 0.00750841, "loss/policy_avg": -0.15323098, "loss/value_avg": 0.31320888, "val/clipfrac_avg": 0.06061186, "policy/entropy_avg": 0.14741188, "val/ratio": 0.99015331, "val/ratio_var": 4.773e-05, "val/num_eos_tokens": 86, "lr": 5.15e-06, "episode": 10240, "epoch": 0.51717172, "global_step/max_steps": "640/1238", "percentage": "51.70%", "elapsed_time": "7h 52m 9s", "remaining_time": "7h 21m 10s"} +{"eps": 0, "objective/kl": 46.53107452, "objective/entropy": -184.58500671, "objective/non_score_reward": -2.32655382, "objective/rlhf_reward": 13.3531332, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00080693, "policy/clipfrac_avg": 0.00431749, "loss/policy_avg": -0.40618008, "loss/value_avg": 0.39824045, "val/clipfrac_avg": 0.20341261, "policy/entropy_avg": 0.15611839, "val/ratio": 0.98970211, "val/ratio_var": 4.783e-05, "val/num_eos_tokens": 56, "lr": 5.13e-06, "episode": 10256, "epoch": 0.5179798, "global_step/max_steps": "641/1238", "percentage": "51.78%", "elapsed_time": "7h 53m 25s", "remaining_time": "7h 20m 55s"} +{"eps": 0, "objective/kl": 32.67811966, "objective/entropy": -92.79855347, "objective/non_score_reward": -1.63390601, "objective/rlhf_reward": 11.48328114, "objective/scores": 13.125, "policy/approxkl_avg": 0.00095978, "policy/clipfrac_avg": 0.01371865, "loss/policy_avg": 0.3516452, "loss/value_avg": 0.76101393, "val/clipfrac_avg": 0.1001372, "policy/entropy_avg": 0.20919991, "val/ratio": 0.98836172, "val/ratio_var": 7.447e-05, "val/num_eos_tokens": 227, "lr": 5.12e-06, "episode": 10272, "epoch": 0.51878788, "global_step/max_steps": "642/1238", "percentage": "51.86%", "elapsed_time": "7h 53m 59s", "remaining_time": "7h 20m 2s"} +{"eps": 0, "objective/kl": 27.62618256, "objective/entropy": -61.30334473, "objective/non_score_reward": -1.38130915, "objective/rlhf_reward": 14.43119049, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00092722, "policy/clipfrac_avg": 0.01159144, "loss/policy_avg": 0.17767911, "loss/value_avg": 0.53784502, "val/clipfrac_avg": 0.08490149, "policy/entropy_avg": 0.17319107, "val/ratio": 0.98952937, "val/ratio_var": 5.064e-05, "val/num_eos_tokens": 201, "lr": 5.11e-06, "episode": 10288, "epoch": 0.51959596, "global_step/max_steps": "643/1238", "percentage": "51.94%", "elapsed_time": "7h 54m 27s", "remaining_time": "7h 19m 2s"} +{"eps": 0, "objective/kl": 42.24378967, "objective/entropy": -100.73757172, "objective/non_score_reward": -2.11218929, "objective/rlhf_reward": 12.33312321, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00086872, "policy/clipfrac_avg": 0.00567754, "loss/policy_avg": -0.00483158, "loss/value_avg": 0.30399805, "val/clipfrac_avg": 0.03101456, "policy/entropy_avg": 0.17821407, "val/ratio": 0.98843867, "val/ratio_var": 4.831e-05, "val/num_eos_tokens": 530, "lr": 5.09e-06, "episode": 10304, "epoch": 0.52040404, "global_step/max_steps": "644/1238", "percentage": "52.02%", "elapsed_time": "7h 55m 17s", "remaining_time": "7h 18m 23s"} +{"eps": 0, "objective/kl": 44.61050415, "objective/entropy": -77.07893372, "objective/non_score_reward": -2.23052526, "objective/rlhf_reward": 10.77728653, "objective/scores": 13.0, "policy/approxkl_avg": 0.00098348, "policy/clipfrac_avg": 0.00576425, "loss/policy_avg": -0.14808293, "loss/value_avg": 1.22150862, "val/clipfrac_avg": 0.06691774, "policy/entropy_avg": 0.20584464, "val/ratio": 0.98759341, "val/ratio_var": 6.56e-05, "val/num_eos_tokens": 44, "lr": 5.08e-06, "episode": 10320, "epoch": 0.52121212, "global_step/max_steps": "645/1238", "percentage": "52.10%", "elapsed_time": "7h 55m 54s", "remaining_time": "7h 17m 32s"} +{"eps": 0, "objective/kl": 37.04098511, "objective/entropy": -102.19164276, "objective/non_score_reward": -1.85204959, "objective/rlhf_reward": 15.03076363, "objective/scores": 16.875, "policy/approxkl_avg": 0.00085761, "policy/clipfrac_avg": 0.00519873, "loss/policy_avg": 0.02113627, "loss/value_avg": 0.24227744, "val/clipfrac_avg": 0.02398884, "policy/entropy_avg": 0.16482449, "val/ratio": 0.98885286, "val/ratio_var": 3.119e-05, "val/num_eos_tokens": 250, "lr": 5.07e-06, "episode": 10336, "epoch": 0.5220202, "global_step/max_steps": "646/1238", "percentage": "52.18%", "elapsed_time": "7h 56m 30s", "remaining_time": "7h 16m 40s"} +{"eps": 0, "objective/kl": 39.35309219, "objective/entropy": -93.60134888, "objective/non_score_reward": -1.96765471, "objective/rlhf_reward": 13.82921982, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00090728, "policy/clipfrac_avg": 0.01092708, "loss/policy_avg": 0.13451204, "loss/value_avg": 0.67601633, "val/clipfrac_avg": 0.10246877, "policy/entropy_avg": 0.16292381, "val/ratio": 0.98879766, "val/ratio_var": 6.221e-05, "val/num_eos_tokens": 159, "lr": 5.05e-06, "episode": 10352, "epoch": 0.52282828, "global_step/max_steps": "647/1238", "percentage": "52.26%", "elapsed_time": "7h 57m 1s", "remaining_time": "7h 15m 44s"} +{"eps": 0, "objective/kl": 45.73390961, "objective/entropy": -218.7485199, "objective/non_score_reward": -2.28669548, "objective/rlhf_reward": 13.22892952, "objective/scores": 15.5, "policy/approxkl_avg": 0.00075758, "policy/clipfrac_avg": 0.0122658, "loss/policy_avg": -0.0343408, "loss/value_avg": 0.26276273, "val/clipfrac_avg": 0.09911302, "policy/entropy_avg": 0.15109158, "val/ratio": 0.98977321, "val/ratio_var": 5.469e-05, "val/num_eos_tokens": 287, "lr": 5.04e-06, "episode": 10368, "epoch": 0.52363636, "global_step/max_steps": "648/1238", "percentage": "52.34%", "elapsed_time": "7h 58m 43s", "remaining_time": "7h 15m 52s"} +{"eps": 0, "objective/kl": 38.24567795, "objective/entropy": -154.6942749, "objective/non_score_reward": -1.9122839, "objective/rlhf_reward": 12.0799036, "objective/scores": 14.0, "policy/approxkl_avg": 0.00078985, "policy/clipfrac_avg": 0.00485107, "loss/policy_avg": 0.17616849, "loss/value_avg": 0.9873184, "val/clipfrac_avg": 0.05088341, "policy/entropy_avg": 0.18099499, "val/ratio": 0.98992676, "val/ratio_var": 4.216e-05, "val/num_eos_tokens": 1593, "lr": 5.03e-06, "episode": 10384, "epoch": 0.52444444, "global_step/max_steps": "649/1238", "percentage": "52.42%", "elapsed_time": "8h 0m 27s", "remaining_time": "7h 16m 1s"} +{"eps": 0, "objective/kl": 27.84048653, "objective/entropy": -90.7487793, "objective/non_score_reward": -1.39202428, "objective/rlhf_reward": 14.35797501, "objective/scores": 15.75, "policy/approxkl_avg": 0.0008531, "policy/clipfrac_avg": 0.00598605, "loss/policy_avg": -0.10312466, "loss/value_avg": 0.73487103, "val/clipfrac_avg": 0.17565612, "policy/entropy_avg": 0.14681435, "val/ratio": 0.98980904, "val/ratio_var": 6.202e-05, "val/num_eos_tokens": 591, "lr": 5.01e-06, "episode": 10400, "epoch": 0.52525253, "global_step/max_steps": "650/1238", "percentage": "52.50%", "elapsed_time": "8h 1m 14s", "remaining_time": "7h 15m 20s"} +{"eps": 0, "objective/kl": 36.00543976, "objective/entropy": -116.00996399, "objective/non_score_reward": -1.80027199, "objective/rlhf_reward": 13.37941551, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00080676, "policy/clipfrac_avg": 0.00669206, "loss/policy_avg": -0.37787753, "loss/value_avg": 0.40722805, "val/clipfrac_avg": 0.12555306, "policy/entropy_avg": 0.15956879, "val/ratio": 0.98895139, "val/ratio_var": 7.937e-05, "val/num_eos_tokens": 475, "lr": 5e-06, "episode": 10416, "epoch": 0.52606061, "global_step/max_steps": "651/1238", "percentage": "52.58%", "elapsed_time": "8h 1m 51s", "remaining_time": "7h 14m 29s"} +{"eps": 0, "objective/kl": 35.39002609, "objective/entropy": -113.37627411, "objective/non_score_reward": -1.76950121, "objective/rlhf_reward": 12.87893677, "objective/scores": 14.625, "policy/approxkl_avg": 0.00090438, "policy/clipfrac_avg": 0.01133544, "loss/policy_avg": -0.02487482, "loss/value_avg": 0.30301523, "val/clipfrac_avg": 0.05399475, "policy/entropy_avg": 0.16724396, "val/ratio": 0.98926604, "val/ratio_var": 4.781e-05, "val/num_eos_tokens": 72, "lr": 4.99e-06, "episode": 10432, "epoch": 0.52686869, "global_step/max_steps": "652/1238", "percentage": "52.67%", "elapsed_time": "8h 2m 36s", "remaining_time": "7h 13m 45s"} +{"eps": 0, "objective/kl": 31.19072914, "objective/entropy": -97.41447449, "objective/non_score_reward": -1.55953646, "objective/rlhf_reward": 14.83890057, "objective/scores": 16.375, "policy/approxkl_avg": 0.00080206, "policy/clipfrac_avg": 0.0035828, "loss/policy_avg": -0.45762679, "loss/value_avg": 0.63106894, "val/clipfrac_avg": 0.17964357, "policy/entropy_avg": 0.1652298, "val/ratio": 0.98922312, "val/ratio_var": 5.022e-05, "val/num_eos_tokens": 25, "lr": 4.97e-06, "episode": 10448, "epoch": 0.52767677, "global_step/max_steps": "653/1238", "percentage": "52.75%", "elapsed_time": "8h 3m 21s", "remaining_time": "7h 13m 1s"} +{"eps": 0, "objective/kl": 33.45720673, "objective/entropy": -157.48797607, "objective/non_score_reward": -1.67286026, "objective/rlhf_reward": 12.96776485, "objective/scores": 14.625, "policy/approxkl_avg": 0.00098673, "policy/clipfrac_avg": 0.00845314, "loss/policy_avg": -0.14918374, "loss/value_avg": 0.73288929, "val/clipfrac_avg": 0.1064778, "policy/entropy_avg": 0.15486336, "val/ratio": 0.98896325, "val/ratio_var": 0.00010885, "val/num_eos_tokens": 1553, "lr": 4.96e-06, "episode": 10464, "epoch": 0.52848485, "global_step/max_steps": "654/1238", "percentage": "52.83%", "elapsed_time": "8h 5m 3s", "remaining_time": "7h 13m 8s"} +{"eps": 0, "objective/kl": 41.44767761, "objective/entropy": -151.5229187, "objective/non_score_reward": -2.07238388, "objective/rlhf_reward": 11.59949112, "objective/scores": 13.6875, "policy/approxkl_avg": 0.00101964, "policy/clipfrac_avg": 0.00850991, "loss/policy_avg": -0.00173752, "loss/value_avg": 0.47492418, "val/clipfrac_avg": 0.1348328, "policy/entropy_avg": 0.17792892, "val/ratio": 0.98858857, "val/ratio_var": 3.869e-05, "val/num_eos_tokens": 723, "lr": 4.95e-06, "episode": 10480, "epoch": 0.52929293, "global_step/max_steps": "655/1238", "percentage": "52.91%", "elapsed_time": "8h 6m 3s", "remaining_time": "7h 12m 37s"} +{"eps": 0, "objective/kl": 37.21573639, "objective/entropy": -98.99279022, "objective/non_score_reward": -1.86078691, "objective/rlhf_reward": 12.45952606, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00089326, "policy/clipfrac_avg": 0.01308158, "loss/policy_avg": 0.30413297, "loss/value_avg": 0.46963644, "val/clipfrac_avg": 0.09068674, "policy/entropy_avg": 0.16152191, "val/ratio": 0.9888199, "val/ratio_var": 6.497e-05, "val/num_eos_tokens": 495, "lr": 4.93e-06, "episode": 10496, "epoch": 0.53010101, "global_step/max_steps": "656/1238", "percentage": "52.99%", "elapsed_time": "8h 6m 35s", "remaining_time": "7h 11m 42s"} +{"eps": 0, "objective/kl": 38.03298187, "objective/entropy": -104.63330078, "objective/non_score_reward": -1.90164924, "objective/rlhf_reward": 12.34835052, "objective/scores": 14.25, "policy/approxkl_avg": 0.00072154, "policy/clipfrac_avg": 0.00387173, "loss/policy_avg": 0.10661557, "loss/value_avg": 1.98906946, "val/clipfrac_avg": 0.17002994, "policy/entropy_avg": 0.14897156, "val/ratio": 0.9906019, "val/ratio_var": 3.762e-05, "val/num_eos_tokens": 9, "lr": 4.92e-06, "episode": 10512, "epoch": 0.53090909, "global_step/max_steps": "657/1238", "percentage": "53.07%", "elapsed_time": "8h 7m 33s", "remaining_time": "7h 11m 9s"} +{"eps": 0, "objective/kl": 36.28617859, "objective/entropy": -83.36598206, "objective/non_score_reward": -1.81430888, "objective/rlhf_reward": 14.56069088, "objective/scores": 16.375, "policy/approxkl_avg": 0.00088938, "policy/clipfrac_avg": 0.011145, "loss/policy_avg": 0.04127348, "loss/value_avg": 0.29201186, "val/clipfrac_avg": 0.03217828, "policy/entropy_avg": 0.16919899, "val/ratio": 0.98925114, "val/ratio_var": 3.13e-05, "val/num_eos_tokens": 377, "lr": 4.91e-06, "episode": 10528, "epoch": 0.53171717, "global_step/max_steps": "658/1238", "percentage": "53.15%", "elapsed_time": "8h 8m 21s", "remaining_time": "7h 10m 28s"} +{"eps": 0, "objective/kl": 32.11946106, "objective/entropy": -46.25899506, "objective/non_score_reward": -1.60597312, "objective/rlhf_reward": 12.77684021, "objective/scores": 14.375, "policy/approxkl_avg": 0.00124343, "policy/clipfrac_avg": 0.00876493, "loss/policy_avg": 0.04267742, "loss/value_avg": 0.28014019, "val/clipfrac_avg": 0.07641617, "policy/entropy_avg": 0.18162346, "val/ratio": 0.98736161, "val/ratio_var": 0.00011701, "val/num_eos_tokens": 16, "lr": 4.89e-06, "episode": 10544, "epoch": 0.53252525, "global_step/max_steps": "659/1238", "percentage": "53.23%", "elapsed_time": "8h 8m 59s", "remaining_time": "7h 9m 38s"} +{"eps": 0, "objective/kl": 39.09260559, "objective/entropy": -151.6408844, "objective/non_score_reward": -1.95463026, "objective/rlhf_reward": 11.2172451, "objective/scores": 13.1875, "policy/approxkl_avg": 0.00079586, "policy/clipfrac_avg": 0.00742696, "loss/policy_avg": 0.20670435, "loss/value_avg": 0.48474213, "val/clipfrac_avg": 0.07210306, "policy/entropy_avg": 0.15951443, "val/ratio": 0.98869824, "val/ratio_var": 4.409e-05, "val/num_eos_tokens": 20, "lr": 4.88e-06, "episode": 10560, "epoch": 0.53333333, "global_step/max_steps": "660/1238", "percentage": "53.31%", "elapsed_time": "8h 10m 40s", "remaining_time": "7h 9m 43s"} +{"eps": 0, "objective/kl": 39.04478073, "objective/entropy": -82.99937439, "objective/non_score_reward": -1.95223916, "objective/rlhf_reward": 14.46182346, "objective/scores": 16.375, "policy/approxkl_avg": 0.00080986, "policy/clipfrac_avg": 0.00760706, "loss/policy_avg": -0.01065885, "loss/value_avg": 0.23517704, "val/clipfrac_avg": 0.13296083, "policy/entropy_avg": 0.15206003, "val/ratio": 0.98953426, "val/ratio_var": 4.896e-05, "val/num_eos_tokens": 369, "lr": 4.87e-06, "episode": 10576, "epoch": 0.53414141, "global_step/max_steps": "661/1238", "percentage": "53.39%", "elapsed_time": "8h 11m 16s", "remaining_time": "7h 8m 50s"} +{"eps": 0, "objective/kl": 33.77119827, "objective/entropy": -50.70814133, "objective/non_score_reward": -1.68856001, "objective/rlhf_reward": 13.56144047, "objective/scores": 15.25, "policy/approxkl_avg": 0.00106827, "policy/clipfrac_avg": 0.00594073, "loss/policy_avg": 0.24839319, "loss/value_avg": 0.38362139, "val/clipfrac_avg": 0.08841181, "policy/entropy_avg": 0.18691349, "val/ratio": 0.98691684, "val/ratio_var": 0.00010278, "val/num_eos_tokens": 336, "lr": 4.85e-06, "episode": 10592, "epoch": 0.53494949, "global_step/max_steps": "662/1238", "percentage": "53.47%", "elapsed_time": "8h 11m 42s", "remaining_time": "7h 7m 49s"} +{"eps": 0, "objective/kl": 40.23773193, "objective/entropy": -22.15380669, "objective/non_score_reward": -2.0118866, "objective/rlhf_reward": 12.6599884, "objective/scores": 14.6875, "policy/approxkl_avg": 0.00119145, "policy/clipfrac_avg": 0.00375203, "loss/policy_avg": 0.01862108, "loss/value_avg": 0.47118768, "val/clipfrac_avg": 0.13863704, "policy/entropy_avg": 0.22987556, "val/ratio": 0.98413563, "val/ratio_var": 8.863e-05, "val/num_eos_tokens": 143, "lr": 4.84e-06, "episode": 10608, "epoch": 0.53575758, "global_step/max_steps": "663/1238", "percentage": "53.55%", "elapsed_time": "8h 12m 13s", "remaining_time": "7h 6m 53s"} +{"eps": 0, "objective/kl": 32.98080444, "objective/entropy": -47.15129852, "objective/non_score_reward": -1.64904022, "objective/rlhf_reward": 12.57752228, "objective/scores": 14.25, "policy/approxkl_avg": 0.00093364, "policy/clipfrac_avg": 0.01062916, "loss/policy_avg": -0.1973286, "loss/value_avg": 0.82946527, "val/clipfrac_avg": 0.04402674, "policy/entropy_avg": 0.17932701, "val/ratio": 0.98854953, "val/ratio_var": 7.598e-05, "val/num_eos_tokens": 230, "lr": 4.83e-06, "episode": 10624, "epoch": 0.53656566, "global_step/max_steps": "664/1238", "percentage": "53.63%", "elapsed_time": "8h 12m 40s", "remaining_time": "7h 5m 53s"} +{"eps": 0, "objective/kl": 35.82035828, "objective/entropy": -92.09422302, "objective/non_score_reward": -1.79101789, "objective/rlhf_reward": 15.99023247, "objective/scores": 17.75, "policy/approxkl_avg": 0.00084645, "policy/clipfrac_avg": 0.005295, "loss/policy_avg": -0.36426514, "loss/value_avg": 0.49642229, "val/clipfrac_avg": 0.18585266, "policy/entropy_avg": 0.16996145, "val/ratio": 0.98797339, "val/ratio_var": 7.157e-05, "val/num_eos_tokens": 760, "lr": 4.81e-06, "episode": 10640, "epoch": 0.53737374, "global_step/max_steps": "665/1238", "percentage": "53.72%", "elapsed_time": "8h 13m 21s", "remaining_time": "7h 5m 5s"} +{"eps": 0, "objective/kl": 36.84406281, "objective/entropy": -100.7431488, "objective/non_score_reward": -1.84220314, "objective/rlhf_reward": 13.72029686, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0008232, "policy/clipfrac_avg": 0.00597512, "loss/policy_avg": 0.19750816, "loss/value_avg": 0.72602856, "val/clipfrac_avg": 0.08335634, "policy/entropy_avg": 0.17266035, "val/ratio": 0.98846883, "val/ratio_var": 5.402e-05, "val/num_eos_tokens": 58, "lr": 4.8e-06, "episode": 10656, "epoch": 0.53818182, "global_step/max_steps": "666/1238", "percentage": "53.80%", "elapsed_time": "8h 13m 56s", "remaining_time": "7h 4m 13s"} +{"eps": 0, "objective/kl": 34.5240097, "objective/entropy": -124.33128357, "objective/non_score_reward": -1.72620058, "objective/rlhf_reward": 11.6409874, "objective/scores": 13.375, "policy/approxkl_avg": 0.00072721, "policy/clipfrac_avg": 0.00860882, "loss/policy_avg": -0.39258122, "loss/value_avg": 0.96636295, "val/clipfrac_avg": 0.03538993, "policy/entropy_avg": 0.14444184, "val/ratio": 0.9909724, "val/ratio_var": 4.301e-05, "val/num_eos_tokens": 439, "lr": 4.79e-06, "episode": 10672, "epoch": 0.5389899, "global_step/max_steps": "667/1238", "percentage": "53.88%", "elapsed_time": "8h 14m 39s", "remaining_time": "7h 3m 27s"} +{"eps": 0, "objective/kl": 35.08434677, "objective/entropy": -34.5688324, "objective/non_score_reward": -1.75421715, "objective/rlhf_reward": 15.20672035, "objective/scores": 17.0, "policy/approxkl_avg": 0.00118859, "policy/clipfrac_avg": 0.00696058, "loss/policy_avg": -0.06382111, "loss/value_avg": 0.39700243, "val/clipfrac_avg": 0.13442066, "policy/entropy_avg": 0.19833374, "val/ratio": 0.98612165, "val/ratio_var": 3.919e-05, "val/num_eos_tokens": 187, "lr": 4.77e-06, "episode": 10688, "epoch": 0.53979798, "global_step/max_steps": "668/1238", "percentage": "53.96%", "elapsed_time": "8h 15m 13s", "remaining_time": "7h 2m 34s"} +{"eps": 0, "objective/kl": 34.1692276, "objective/entropy": -87.38048553, "objective/non_score_reward": -1.70846128, "objective/rlhf_reward": 12.05716324, "objective/scores": 13.75, "policy/approxkl_avg": 0.00091873, "policy/clipfrac_avg": 0.00558793, "loss/policy_avg": 0.16306788, "loss/value_avg": 0.45442262, "val/clipfrac_avg": 0.19067705, "policy/entropy_avg": 0.17048359, "val/ratio": 0.98888832, "val/ratio_var": 6.463e-05, "val/num_eos_tokens": 373, "lr": 4.76e-06, "episode": 10704, "epoch": 0.54060606, "global_step/max_steps": "669/1238", "percentage": "54.04%", "elapsed_time": "8h 16m 1s", "remaining_time": "7h 1m 52s"} +{"eps": 0, "objective/kl": 36.53352356, "objective/entropy": -84.4004364, "objective/non_score_reward": -1.82667601, "objective/rlhf_reward": 13.68113708, "objective/scores": 15.5, "policy/approxkl_avg": 0.00091383, "policy/clipfrac_avg": 0.00606238, "loss/policy_avg": 0.01032647, "loss/value_avg": 0.34742054, "val/clipfrac_avg": 0.09507844, "policy/entropy_avg": 0.17022514, "val/ratio": 0.98813802, "val/ratio_var": 4.751e-05, "val/num_eos_tokens": 295, "lr": 4.75e-06, "episode": 10720, "epoch": 0.54141414, "global_step/max_steps": "670/1238", "percentage": "54.12%", "elapsed_time": "8h 16m 37s", "remaining_time": "7h 1m 1s"} +{"eps": 0, "objective/kl": 39.16799164, "objective/entropy": -81.22120667, "objective/non_score_reward": -1.95839989, "objective/rlhf_reward": 13.49472427, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00090189, "policy/clipfrac_avg": 0.01516369, "loss/policy_avg": -0.09906018, "loss/value_avg": 0.80662519, "val/clipfrac_avg": 0.0449674, "policy/entropy_avg": 0.17831039, "val/ratio": 0.98862517, "val/ratio_var": 7.143e-05, "val/num_eos_tokens": 582, "lr": 4.73e-06, "episode": 10736, "epoch": 0.54222222, "global_step/max_steps": "671/1238", "percentage": "54.20%", "elapsed_time": "8h 17m 17s", "remaining_time": "7h 0m 12s"} +{"eps": 0, "objective/kl": 37.0121994, "objective/entropy": -144.1421814, "objective/non_score_reward": -1.85060978, "objective/rlhf_reward": 14.22751427, "objective/scores": 16.125, "policy/approxkl_avg": 0.00089229, "policy/clipfrac_avg": 0.00648347, "loss/policy_avg": -0.22061354, "loss/value_avg": 0.23615602, "val/clipfrac_avg": 0.07687856, "policy/entropy_avg": 0.17594242, "val/ratio": 0.98865372, "val/ratio_var": 6.715e-05, "val/num_eos_tokens": 513, "lr": 4.72e-06, "episode": 10752, "epoch": 0.5430303, "global_step/max_steps": "672/1238", "percentage": "54.28%", "elapsed_time": "8h 18m 8s", "remaining_time": "6h 59m 33s"} +{"eps": 0, "objective/kl": 39.09211349, "objective/entropy": -88.67999268, "objective/non_score_reward": -1.95460558, "objective/rlhf_reward": 13.68601894, "objective/scores": 15.625, "policy/approxkl_avg": 0.00100674, "policy/clipfrac_avg": 0.00690817, "loss/policy_avg": -0.0776338, "loss/value_avg": 0.67938197, "val/clipfrac_avg": 0.077333, "policy/entropy_avg": 0.1997242, "val/ratio": 0.98732162, "val/ratio_var": 6.026e-05, "val/num_eos_tokens": 119, "lr": 4.71e-06, "episode": 10768, "epoch": 0.54383838, "global_step/max_steps": "673/1238", "percentage": "54.36%", "elapsed_time": "8h 18m 43s", "remaining_time": "6h 58m 41s"} +{"eps": 0, "objective/kl": 30.96477318, "objective/entropy": -60.3192482, "objective/non_score_reward": -1.54823875, "objective/rlhf_reward": 12.93613625, "objective/scores": 14.5, "policy/approxkl_avg": 0.00093576, "policy/clipfrac_avg": 0.01179488, "loss/policy_avg": 0.16710958, "loss/value_avg": 0.79779077, "val/clipfrac_avg": 0.14395142, "policy/entropy_avg": 0.17830944, "val/ratio": 0.98842996, "val/ratio_var": 9.556e-05, "val/num_eos_tokens": 346, "lr": 4.69e-06, "episode": 10784, "epoch": 0.54464646, "global_step/max_steps": "674/1238", "percentage": "54.44%", "elapsed_time": "8h 19m 23s", "remaining_time": "6h 57m 53s"} +{"eps": 0, "objective/kl": 32.99176788, "objective/entropy": -66.31764984, "objective/non_score_reward": -1.64958858, "objective/rlhf_reward": 14.38166142, "objective/scores": 16.0, "policy/approxkl_avg": 0.00079042, "policy/clipfrac_avg": 0.00452361, "loss/policy_avg": -0.02382121, "loss/value_avg": 0.22992632, "val/clipfrac_avg": 0.0528938, "policy/entropy_avg": 0.16163445, "val/ratio": 0.98959821, "val/ratio_var": 3.757e-05, "val/num_eos_tokens": 387, "lr": 4.68e-06, "episode": 10800, "epoch": 0.54545455, "global_step/max_steps": "675/1238", "percentage": "54.52%", "elapsed_time": "8h 19m 59s", "remaining_time": "6h 57m 1s"} +{"eps": 0, "objective/kl": 34.93087387, "objective/entropy": -1.3176899, "objective/non_score_reward": -1.74654365, "objective/rlhf_reward": 14.09720612, "objective/scores": 15.875, "policy/approxkl_avg": 0.00111605, "policy/clipfrac_avg": 0.00656351, "loss/policy_avg": 0.0298745, "loss/value_avg": 0.32752854, "val/clipfrac_avg": 0.10146623, "policy/entropy_avg": 0.20054245, "val/ratio": 0.9860273, "val/ratio_var": 3.469e-05, "val/num_eos_tokens": 224, "lr": 4.67e-06, "episode": 10816, "epoch": 0.54626263, "global_step/max_steps": "676/1238", "percentage": "54.60%", "elapsed_time": "8h 20m 41s", "remaining_time": "6h 56m 14s"} +{"eps": 0, "objective/kl": 36.15802383, "objective/entropy": -96.90899658, "objective/non_score_reward": -1.80790126, "objective/rlhf_reward": 13.32491112, "objective/scores": 15.125, "policy/approxkl_avg": 0.00087181, "policy/clipfrac_avg": 0.00934893, "loss/policy_avg": -0.10157569, "loss/value_avg": 0.49072486, "val/clipfrac_avg": 0.02000356, "policy/entropy_avg": 0.1555233, "val/ratio": 0.98968899, "val/ratio_var": 5.075e-05, "val/num_eos_tokens": 119, "lr": 4.65e-06, "episode": 10832, "epoch": 0.54707071, "global_step/max_steps": "677/1238", "percentage": "54.68%", "elapsed_time": "8h 21m 20s", "remaining_time": "6h 55m 26s"} +{"eps": 0, "objective/kl": 31.89101219, "objective/entropy": -82.16497803, "objective/non_score_reward": -1.59455073, "objective/rlhf_reward": 15.30388641, "objective/scores": 16.875, "policy/approxkl_avg": 0.00113607, "policy/clipfrac_avg": 0.00726479, "loss/policy_avg": -0.26314047, "loss/value_avg": 0.38394535, "val/clipfrac_avg": 0.09376705, "policy/entropy_avg": 0.19282532, "val/ratio": 0.98777592, "val/ratio_var": 5.525e-05, "val/num_eos_tokens": 803, "lr": 4.64e-06, "episode": 10848, "epoch": 0.54787879, "global_step/max_steps": "678/1238", "percentage": "54.77%", "elapsed_time": "8h 22m 5s", "remaining_time": "6h 54m 42s"} +{"eps": 0, "objective/kl": 37.95676422, "objective/entropy": -66.71647644, "objective/non_score_reward": -1.89783835, "objective/rlhf_reward": 14.82091141, "objective/scores": 16.75, "policy/approxkl_avg": 0.00094471, "policy/clipfrac_avg": 0.00472238, "loss/policy_avg": -0.25290811, "loss/value_avg": 1.24626839, "val/clipfrac_avg": 0.15111136, "policy/entropy_avg": 0.16110456, "val/ratio": 0.9897334, "val/ratio_var": 5.177e-05, "val/num_eos_tokens": 376, "lr": 4.63e-06, "episode": 10864, "epoch": 0.54868687, "global_step/max_steps": "679/1238", "percentage": "54.85%", "elapsed_time": "8h 22m 38s", "remaining_time": "6h 53m 48s"} +{"eps": 0, "objective/kl": 35.02674103, "objective/entropy": -96.83143616, "objective/non_score_reward": -1.75133705, "objective/rlhf_reward": 11.63147545, "objective/scores": 13.375, "policy/approxkl_avg": 0.00080161, "policy/clipfrac_avg": 0.00808819, "loss/policy_avg": 0.26044035, "loss/value_avg": 0.96192902, "val/clipfrac_avg": 0.18738833, "policy/entropy_avg": 0.14316559, "val/ratio": 0.99032533, "val/ratio_var": 3.636e-05, "val/num_eos_tokens": 55, "lr": 4.61e-06, "episode": 10880, "epoch": 0.54949495, "global_step/max_steps": "680/1238", "percentage": "54.93%", "elapsed_time": "8h 23m 17s", "remaining_time": "6h 52m 59s"} +{"eps": 0, "objective/kl": 35.87485886, "objective/entropy": -46.69654846, "objective/non_score_reward": -1.7937429, "objective/rlhf_reward": 12.95625687, "objective/scores": 14.75, "policy/approxkl_avg": 0.00111595, "policy/clipfrac_avg": 0.0070981, "loss/policy_avg": 0.20710929, "loss/value_avg": 0.82125533, "val/clipfrac_avg": 0.10262697, "policy/entropy_avg": 0.18874073, "val/ratio": 0.98705566, "val/ratio_var": 4.734e-05, "val/num_eos_tokens": 32, "lr": 4.6e-06, "episode": 10896, "epoch": 0.55030303, "global_step/max_steps": "681/1238", "percentage": "55.01%", "elapsed_time": "8h 24m 7s", "remaining_time": "6h 52m 20s"} +{"eps": 0, "objective/kl": 45.23626709, "objective/entropy": -150.32344055, "objective/non_score_reward": -2.2618134, "objective/rlhf_reward": 12.62881088, "objective/scores": 14.875, "policy/approxkl_avg": 0.00131207, "policy/clipfrac_avg": 0.00512622, "loss/policy_avg": -0.1979555, "loss/value_avg": 0.3630096, "val/clipfrac_avg": 0.093025, "policy/entropy_avg": 0.15036583, "val/ratio": 0.98999137, "val/ratio_var": 5.336e-05, "val/num_eos_tokens": 451, "lr": 4.59e-06, "episode": 10912, "epoch": 0.55111111, "global_step/max_steps": "682/1238", "percentage": "55.09%", "elapsed_time": "8h 24m 53s", "remaining_time": "6h 51m 37s"} +{"eps": 0, "objective/kl": 31.69464111, "objective/entropy": -142.62588501, "objective/non_score_reward": -1.58473217, "objective/rlhf_reward": 14.21214294, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00073179, "policy/clipfrac_avg": 0.01235208, "loss/policy_avg": 0.18228947, "loss/value_avg": 0.86032301, "val/clipfrac_avg": 0.14113879, "policy/entropy_avg": 0.13785887, "val/ratio": 0.99092472, "val/ratio_var": 5.325e-05, "val/num_eos_tokens": 289, "lr": 4.57e-06, "episode": 10928, "epoch": 0.55191919, "global_step/max_steps": "683/1238", "percentage": "55.17%", "elapsed_time": "8h 25m 37s", "remaining_time": "6h 50m 52s"} +{"eps": 0, "objective/kl": 43.83638, "objective/entropy": -27.27905273, "objective/non_score_reward": -2.19181919, "objective/rlhf_reward": 14.43318081, "objective/scores": 16.625, "policy/approxkl_avg": 0.00109796, "policy/clipfrac_avg": 0.00580876, "loss/policy_avg": -0.0519242, "loss/value_avg": 0.19368619, "val/clipfrac_avg": 0.00993679, "policy/entropy_avg": 0.21721268, "val/ratio": 0.98618066, "val/ratio_var": 3.975e-05, "val/num_eos_tokens": 24, "lr": 4.56e-06, "episode": 10944, "epoch": 0.55272727, "global_step/max_steps": "684/1238", "percentage": "55.25%", "elapsed_time": "8h 26m 12s", "remaining_time": "6h 49m 59s"} +{"eps": 0, "objective/kl": 26.13726425, "objective/entropy": -114.47680664, "objective/non_score_reward": -1.30686331, "objective/rlhf_reward": 12.94313717, "objective/scores": 14.25, "policy/approxkl_avg": 0.0007893, "policy/clipfrac_avg": 0.00762468, "loss/policy_avg": 0.11247897, "loss/value_avg": 0.36871308, "val/clipfrac_avg": 0.08647881, "policy/entropy_avg": 0.14629364, "val/ratio": 0.9891907, "val/ratio_var": 7.146e-05, "val/num_eos_tokens": 446, "lr": 4.55e-06, "episode": 10960, "epoch": 0.55353535, "global_step/max_steps": "685/1238", "percentage": "55.33%", "elapsed_time": "8h 26m 52s", "remaining_time": "6h 49m 11s"} +{"eps": 0, "objective/kl": 28.20573425, "objective/entropy": -129.17437744, "objective/non_score_reward": -1.41028666, "objective/rlhf_reward": 11.9881506, "objective/scores": 13.375, "policy/approxkl_avg": 0.00079815, "policy/clipfrac_avg": 0.0044038, "loss/policy_avg": -0.03544383, "loss/value_avg": 0.3387453, "val/clipfrac_avg": 0.0560783, "policy/entropy_avg": 0.15001488, "val/ratio": 0.98969793, "val/ratio_var": 4.44e-05, "val/num_eos_tokens": 331, "lr": 4.53e-06, "episode": 10976, "epoch": 0.55434343, "global_step/max_steps": "686/1238", "percentage": "55.41%", "elapsed_time": "8h 27m 42s", "remaining_time": "6h 48m 32s"} +{"eps": 0, "objective/kl": 34.03411865, "objective/entropy": -69.12667084, "objective/non_score_reward": -1.70170617, "objective/rlhf_reward": 13.39985657, "objective/scores": 15.125, "policy/approxkl_avg": 0.0009728, "policy/clipfrac_avg": 0.00573809, "loss/policy_avg": 0.03433602, "loss/value_avg": 0.79302192, "val/clipfrac_avg": 0.11831801, "policy/entropy_avg": 0.19457245, "val/ratio": 0.9865483, "val/ratio_var": 7.134e-05, "val/num_eos_tokens": 44, "lr": 4.52e-06, "episode": 10992, "epoch": 0.55515152, "global_step/max_steps": "687/1238", "percentage": "55.49%", "elapsed_time": "8h 28m 30s", "remaining_time": "6h 47m 50s"} +{"eps": 0, "objective/kl": 30.15737534, "objective/entropy": -71.97180939, "objective/non_score_reward": -1.50786877, "objective/rlhf_reward": 13.20306969, "objective/scores": 14.6875, "policy/approxkl_avg": 0.00087443, "policy/clipfrac_avg": 0.00509373, "loss/policy_avg": -0.22896869, "loss/value_avg": 0.90525615, "val/clipfrac_avg": 0.08035694, "policy/entropy_avg": 0.15955305, "val/ratio": 0.98893321, "val/ratio_var": 3.424e-05, "val/num_eos_tokens": 290, "lr": 4.51e-06, "episode": 11008, "epoch": 0.5559596, "global_step/max_steps": "688/1238", "percentage": "55.57%", "elapsed_time": "8h 29m 5s", "remaining_time": "6h 46m 58s"} +{"eps": 0, "objective/kl": 33.09301376, "objective/entropy": -78.77761078, "objective/non_score_reward": -1.65465081, "objective/rlhf_reward": 14.28284931, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0009888, "policy/clipfrac_avg": 0.00978151, "loss/policy_avg": 0.03792828, "loss/value_avg": 0.91238451, "val/clipfrac_avg": 0.07123228, "policy/entropy_avg": 0.17550278, "val/ratio": 0.98792243, "val/ratio_var": 6.044e-05, "val/num_eos_tokens": 16, "lr": 4.49e-06, "episode": 11024, "epoch": 0.55676768, "global_step/max_steps": "689/1238", "percentage": "55.65%", "elapsed_time": "8h 29m 38s", "remaining_time": "6h 46m 4s"} +{"eps": 0, "objective/kl": 33.34143829, "objective/entropy": 5.36783361, "objective/non_score_reward": -1.66707206, "objective/rlhf_reward": 13.6219902, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00157787, "policy/clipfrac_avg": 0.00926361, "loss/policy_avg": 0.01553966, "loss/value_avg": 0.38361928, "val/clipfrac_avg": 0.05502687, "policy/entropy_avg": 0.28288269, "val/ratio": 0.98184955, "val/ratio_var": 7.278e-05, "val/num_eos_tokens": 182, "lr": 4.48e-06, "episode": 11040, "epoch": 0.55757576, "global_step/max_steps": "690/1238", "percentage": "55.74%", "elapsed_time": "8h 30m 12s", "remaining_time": "6h 45m 12s"} +{"eps": 0, "objective/kl": 33.73709106, "objective/entropy": -61.70869064, "objective/non_score_reward": -1.6868546, "objective/rlhf_reward": 14.25845718, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00087692, "policy/clipfrac_avg": 0.00522747, "loss/policy_avg": -0.05112517, "loss/value_avg": 0.38562021, "val/clipfrac_avg": 0.03203329, "policy/entropy_avg": 0.17685509, "val/ratio": 0.98876143, "val/ratio_var": 4.992e-05, "val/num_eos_tokens": 406, "lr": 4.47e-06, "episode": 11056, "epoch": 0.55838384, "global_step/max_steps": "691/1238", "percentage": "55.82%", "elapsed_time": "8h 30m 48s", "remaining_time": "6h 44m 21s"} +{"eps": 0, "objective/kl": 31.42512131, "objective/entropy": -69.50019836, "objective/non_score_reward": -1.57125616, "objective/rlhf_reward": 12.92093086, "objective/scores": 14.5, "policy/approxkl_avg": 0.00096559, "policy/clipfrac_avg": 0.00900004, "loss/policy_avg": 0.29204708, "loss/value_avg": 0.57525468, "val/clipfrac_avg": 0.0944497, "policy/entropy_avg": 0.18520164, "val/ratio": 0.98675066, "val/ratio_var": 9.782e-05, "val/num_eos_tokens": 42, "lr": 4.45e-06, "episode": 11072, "epoch": 0.55919192, "global_step/max_steps": "692/1238", "percentage": "55.90%", "elapsed_time": "8h 31m 28s", "remaining_time": "6h 43m 33s"} +{"eps": 0, "objective/kl": 31.97864151, "objective/entropy": -95.70045471, "objective/non_score_reward": -1.59893203, "objective/rlhf_reward": 13.79169273, "objective/scores": 15.375, "policy/approxkl_avg": 0.0008498, "policy/clipfrac_avg": 0.01487479, "loss/policy_avg": -0.30498353, "loss/value_avg": 0.43966362, "val/clipfrac_avg": 0.08384636, "policy/entropy_avg": 0.15596485, "val/ratio": 0.98999369, "val/ratio_var": 7.268e-05, "val/num_eos_tokens": 116, "lr": 4.44e-06, "episode": 11088, "epoch": 0.56, "global_step/max_steps": "693/1238", "percentage": "55.98%", "elapsed_time": "8h 32m 12s", "remaining_time": "6h 42m 49s"} +{"eps": 0, "objective/kl": 29.76748466, "objective/entropy": -68.41473389, "objective/non_score_reward": -1.48837423, "objective/rlhf_reward": 12.29287624, "objective/scores": 13.75, "policy/approxkl_avg": 0.00101676, "policy/clipfrac_avg": 0.01529287, "loss/policy_avg": -0.28100529, "loss/value_avg": 0.36995184, "val/clipfrac_avg": 0.14090893, "policy/entropy_avg": 0.18943501, "val/ratio": 0.98824054, "val/ratio_var": 9.476e-05, "val/num_eos_tokens": 18, "lr": 4.43e-06, "episode": 11104, "epoch": 0.56080808, "global_step/max_steps": "694/1238", "percentage": "56.06%", "elapsed_time": "8h 32m 53s", "remaining_time": "6h 42m 2s"} +{"eps": 0, "objective/kl": 32.63812256, "objective/entropy": -12.01763153, "objective/non_score_reward": -1.63190615, "objective/rlhf_reward": 15.24309349, "objective/scores": 16.875, "policy/approxkl_avg": 0.00100999, "policy/clipfrac_avg": 0.00408468, "loss/policy_avg": -0.06622212, "loss/value_avg": 0.33488637, "val/clipfrac_avg": 0.06232115, "policy/entropy_avg": 0.18613625, "val/ratio": 0.98712742, "val/ratio_var": 4.299e-05, "val/num_eos_tokens": 267, "lr": 4.41e-06, "episode": 11120, "epoch": 0.56161616, "global_step/max_steps": "695/1238", "percentage": "56.14%", "elapsed_time": "8h 33m 29s", "remaining_time": "6h 41m 11s"} +{"eps": 0, "objective/kl": 28.68563271, "objective/entropy": -87.83956909, "objective/non_score_reward": -1.43428171, "objective/rlhf_reward": 11.4172802, "objective/scores": 12.875, "policy/approxkl_avg": 0.00087592, "policy/clipfrac_avg": 0.00676766, "loss/policy_avg": -0.06522264, "loss/value_avg": 0.71035802, "val/clipfrac_avg": 0.18274252, "policy/entropy_avg": 0.19121313, "val/ratio": 0.98971856, "val/ratio_var": 5.893e-05, "val/num_eos_tokens": 573, "lr": 4.4e-06, "episode": 11136, "epoch": 0.56242424, "global_step/max_steps": "696/1238", "percentage": "56.22%", "elapsed_time": "8h 34m 7s", "remaining_time": "6h 40m 21s"} +{"eps": 0, "objective/kl": 33.1584816, "objective/entropy": -99.48849487, "objective/non_score_reward": -1.65792418, "objective/rlhf_reward": 11.59207535, "objective/scores": 13.25, "policy/approxkl_avg": 0.0008882, "policy/clipfrac_avg": 0.00762211, "loss/policy_avg": 0.17792208, "loss/value_avg": 0.694821, "val/clipfrac_avg": 0.13009925, "policy/entropy_avg": 0.16399193, "val/ratio": 0.98940963, "val/ratio_var": 4.734e-05, "val/num_eos_tokens": 59, "lr": 4.39e-06, "episode": 11152, "epoch": 0.56323232, "global_step/max_steps": "697/1238", "percentage": "56.30%", "elapsed_time": "8h 34m 47s", "remaining_time": "6h 39m 34s"} +{"eps": 0, "objective/kl": 28.16010094, "objective/entropy": -121.60695648, "objective/non_score_reward": -1.40800512, "objective/rlhf_reward": 14.02168274, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00078137, "policy/clipfrac_avg": 0.010616, "loss/policy_avg": 0.22691625, "loss/value_avg": 0.51836574, "val/clipfrac_avg": 0.04978543, "policy/entropy_avg": 0.14654422, "val/ratio": 0.99061865, "val/ratio_var": 7.28e-05, "val/num_eos_tokens": 567, "lr": 4.37e-06, "episode": 11168, "epoch": 0.5640404, "global_step/max_steps": "698/1238", "percentage": "56.38%", "elapsed_time": "8h 35m 23s", "remaining_time": "6h 38m 43s"} +{"eps": 0, "objective/kl": 29.71828651, "objective/entropy": -56.37757111, "objective/non_score_reward": -1.48591447, "objective/rlhf_reward": 14.08439827, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0009621, "policy/clipfrac_avg": 0.01159492, "loss/policy_avg": 0.47537932, "loss/value_avg": 0.88969344, "val/clipfrac_avg": 0.02892506, "policy/entropy_avg": 0.15114403, "val/ratio": 0.98860693, "val/ratio_var": 5.033e-05, "val/num_eos_tokens": 93, "lr": 4.36e-06, "episode": 11184, "epoch": 0.56484848, "global_step/max_steps": "699/1238", "percentage": "56.46%", "elapsed_time": "8h 35m 53s", "remaining_time": "6h 37m 47s"} +{"eps": 0, "objective/kl": 30.32842445, "objective/entropy": -51.71789932, "objective/non_score_reward": -1.51642132, "objective/rlhf_reward": 11.10857868, "objective/scores": 12.625, "policy/approxkl_avg": 0.00109087, "policy/clipfrac_avg": 0.00506787, "loss/policy_avg": 0.01362741, "loss/value_avg": 0.39067256, "val/clipfrac_avg": 0.05506103, "policy/entropy_avg": 0.21241379, "val/ratio": 0.98561716, "val/ratio_var": 8.689e-05, "val/num_eos_tokens": 84, "lr": 4.35e-06, "episode": 11200, "epoch": 0.56565657, "global_step/max_steps": "700/1238", "percentage": "56.54%", "elapsed_time": "8h 36m 22s", "remaining_time": "6h 36m 52s"} +{"eps": 0, "objective/kl": 33.42132568, "objective/entropy": -74.9574585, "objective/non_score_reward": -1.67106628, "objective/rlhf_reward": 14.82893372, "objective/scores": 16.5, "policy/approxkl_avg": 0.00131845, "policy/clipfrac_avg": 0.00809475, "loss/policy_avg": -0.09055965, "loss/value_avg": 0.54550719, "val/clipfrac_avg": 0.18833743, "policy/entropy_avg": 0.17902374, "val/ratio": 0.98792624, "val/ratio_var": 6.12e-05, "val/num_eos_tokens": 358, "lr": 4.33e-06, "episode": 11216, "epoch": 0.56646465, "global_step/max_steps": "701/1238", "percentage": "56.62%", "elapsed_time": "8h 37m 21s", "remaining_time": "6h 36m 19s"} +{"eps": 0, "objective/kl": 27.53748512, "objective/entropy": -90.6164856, "objective/non_score_reward": -1.37687445, "objective/rlhf_reward": 12.82625008, "objective/scores": 14.1875, "policy/approxkl_avg": 0.00095362, "policy/clipfrac_avg": 0.01545755, "loss/policy_avg": 0.22614518, "loss/value_avg": 1.54030466, "val/clipfrac_avg": 0.03345062, "policy/entropy_avg": 0.2056222, "val/ratio": 0.98734403, "val/ratio_var": 7.614e-05, "val/num_eos_tokens": 595, "lr": 4.32e-06, "episode": 11232, "epoch": 0.56727273, "global_step/max_steps": "702/1238", "percentage": "56.70%", "elapsed_time": "8h 37m 53s", "remaining_time": "6h 35m 25s"} +{"eps": 0, "objective/kl": 43.21654892, "objective/entropy": -56.11965942, "objective/non_score_reward": -2.1608274, "objective/rlhf_reward": 15.80792236, "objective/scores": 18.0, "policy/approxkl_avg": 0.00106827, "policy/clipfrac_avg": 0.0051146, "loss/policy_avg": -0.05153675, "loss/value_avg": 0.18954748, "val/clipfrac_avg": 0.12178023, "policy/entropy_avg": 0.20781708, "val/ratio": 0.98607075, "val/ratio_var": 5.393e-05, "val/num_eos_tokens": 244, "lr": 4.31e-06, "episode": 11248, "epoch": 0.56808081, "global_step/max_steps": "703/1238", "percentage": "56.79%", "elapsed_time": "8h 38m 31s", "remaining_time": "6h 34m 36s"} +{"eps": 0, "objective/kl": 32.27890396, "objective/entropy": -180.00924683, "objective/non_score_reward": -1.61394513, "objective/rlhf_reward": 12.42511749, "objective/scores": 14.0625, "policy/approxkl_avg": 0.00073898, "policy/clipfrac_avg": 0.00613048, "loss/policy_avg": 0.22105137, "loss/value_avg": 0.43452761, "val/clipfrac_avg": 0.12273524, "policy/entropy_avg": 0.15887287, "val/ratio": 0.99002528, "val/ratio_var": 6.186e-05, "val/num_eos_tokens": 611, "lr": 4.29e-06, "episode": 11264, "epoch": 0.56888889, "global_step/max_steps": "704/1238", "percentage": "56.87%", "elapsed_time": "8h 39m 22s", "remaining_time": "6h 33m 57s"} +{"eps": 0, "objective/kl": 29.20731735, "objective/entropy": -73.2503891, "objective/non_score_reward": -1.46036589, "objective/rlhf_reward": 15.7661953, "objective/scores": 17.25, "policy/approxkl_avg": 0.00102281, "policy/clipfrac_avg": 0.01073135, "loss/policy_avg": -0.04402062, "loss/value_avg": 1.4599694, "val/clipfrac_avg": 0.1418685, "policy/entropy_avg": 0.17015648, "val/ratio": 0.98653609, "val/ratio_var": 0.0001156, "val/num_eos_tokens": 378, "lr": 4.28e-06, "episode": 11280, "epoch": 0.56969697, "global_step/max_steps": "705/1238", "percentage": "56.95%", "elapsed_time": "8h 39m 58s", "remaining_time": "6h 33m 7s"} +{"eps": 0, "objective/kl": 39.32926178, "objective/entropy": -91.05632019, "objective/non_score_reward": -1.96646333, "objective/rlhf_reward": 12.74447441, "objective/scores": 14.6875, "policy/approxkl_avg": 0.00106001, "policy/clipfrac_avg": 0.00578603, "loss/policy_avg": 0.10734501, "loss/value_avg": 0.40052736, "val/clipfrac_avg": 0.03250089, "policy/entropy_avg": 0.23768234, "val/ratio": 0.9860943, "val/ratio_var": 5.682e-05, "val/num_eos_tokens": 203, "lr": 4.27e-06, "episode": 11296, "epoch": 0.57050505, "global_step/max_steps": "706/1238", "percentage": "57.03%", "elapsed_time": "8h 41m 10s", "remaining_time": "6h 32m 43s"} +{"eps": 0, "objective/kl": 34.54013062, "objective/entropy": -93.25354004, "objective/non_score_reward": -1.72700644, "objective/rlhf_reward": 14.78861809, "objective/scores": 16.5, "policy/approxkl_avg": 0.00084752, "policy/clipfrac_avg": 0.02702557, "loss/policy_avg": -0.50628376, "loss/value_avg": 0.77424705, "val/clipfrac_avg": 0.17750309, "policy/entropy_avg": 0.21597672, "val/ratio": 0.98943341, "val/ratio_var": 9.223e-05, "val/num_eos_tokens": 165, "lr": 4.25e-06, "episode": 11312, "epoch": 0.57131313, "global_step/max_steps": "707/1238", "percentage": "57.11%", "elapsed_time": "8h 41m 43s", "remaining_time": "6h 31m 50s"} +{"eps": 0, "objective/kl": 36.62909698, "objective/entropy": -110.37503052, "objective/non_score_reward": -1.83145487, "objective/rlhf_reward": 12.89510727, "objective/scores": 14.75, "policy/approxkl_avg": 0.00092516, "policy/clipfrac_avg": 0.00941617, "loss/policy_avg": -0.14636691, "loss/value_avg": 0.56326866, "val/clipfrac_avg": 0.0451469, "policy/entropy_avg": 0.16801167, "val/ratio": 0.98972392, "val/ratio_var": 7.692e-05, "val/num_eos_tokens": 267, "lr": 4.24e-06, "episode": 11328, "epoch": 0.57212121, "global_step/max_steps": "708/1238", "percentage": "57.19%", "elapsed_time": "8h 42m 31s", "remaining_time": "6h 31m 9s"} +{"eps": 0, "objective/kl": 31.90000534, "objective/entropy": -76.27204132, "objective/non_score_reward": -1.59500027, "objective/rlhf_reward": 14.05343628, "objective/scores": 15.625, "policy/approxkl_avg": 0.00100204, "policy/clipfrac_avg": 0.01150762, "loss/policy_avg": -0.3529557, "loss/value_avg": 0.70355833, "val/clipfrac_avg": 0.1250823, "policy/entropy_avg": 0.16324425, "val/ratio": 0.98969829, "val/ratio_var": 4.972e-05, "val/num_eos_tokens": 173, "lr": 4.23e-06, "episode": 11344, "epoch": 0.57292929, "global_step/max_steps": "709/1238", "percentage": "57.27%", "elapsed_time": "8h 43m 1s", "remaining_time": "6h 30m 14s"} +{"eps": 0, "objective/kl": 36.81852722, "objective/entropy": -79.55085754, "objective/non_score_reward": -1.84092665, "objective/rlhf_reward": 12.15907383, "objective/scores": 14.0, "policy/approxkl_avg": 0.00110808, "policy/clipfrac_avg": 0.00650397, "loss/policy_avg": 0.16040477, "loss/value_avg": 1.19149637, "val/clipfrac_avg": 0.08140315, "policy/entropy_avg": 0.19612885, "val/ratio": 0.9866147, "val/ratio_var": 7.869e-05, "val/num_eos_tokens": 538, "lr": 4.22e-06, "episode": 11360, "epoch": 0.57373737, "global_step/max_steps": "710/1238", "percentage": "57.35%", "elapsed_time": "8h 43m 35s", "remaining_time": "6h 29m 22s"} +{"eps": 0, "objective/kl": 34.31198502, "objective/entropy": -119.03164673, "objective/non_score_reward": -1.7155993, "objective/rlhf_reward": 13.50315094, "objective/scores": 15.25, "policy/approxkl_avg": 0.00073622, "policy/clipfrac_avg": 0.00589254, "loss/policy_avg": 0.07548796, "loss/value_avg": 0.51380134, "val/clipfrac_avg": 0.02668866, "policy/entropy_avg": 0.14674091, "val/ratio": 0.99014628, "val/ratio_var": 3.967e-05, "val/num_eos_tokens": 515, "lr": 4.2e-06, "episode": 11376, "epoch": 0.57454545, "global_step/max_steps": "711/1238", "percentage": "57.43%", "elapsed_time": "8h 44m 20s", "remaining_time": "6h 28m 38s"} +{"eps": 0, "objective/kl": 38.00650024, "objective/entropy": -112.06816101, "objective/non_score_reward": -1.90032506, "objective/rlhf_reward": 13.99030018, "objective/scores": 15.875, "policy/approxkl_avg": 0.00087484, "policy/clipfrac_avg": 0.00754438, "loss/policy_avg": -0.29259861, "loss/value_avg": 0.38214156, "val/clipfrac_avg": 0.02515335, "policy/entropy_avg": 0.16875196, "val/ratio": 0.98938978, "val/ratio_var": 8.519e-05, "val/num_eos_tokens": 4, "lr": 4.19e-06, "episode": 11392, "epoch": 0.57535354, "global_step/max_steps": "712/1238", "percentage": "57.51%", "elapsed_time": "8h 44m 56s", "remaining_time": "6h 27m 48s"} +{"eps": 0, "objective/kl": 35.69511414, "objective/entropy": -92.28114319, "objective/non_score_reward": -1.78475571, "objective/rlhf_reward": 13.73868179, "objective/scores": 15.5, "policy/approxkl_avg": 0.00084412, "policy/clipfrac_avg": 0.00567072, "loss/policy_avg": -0.54446054, "loss/value_avg": 1.51747489, "val/clipfrac_avg": 0.17910257, "policy/entropy_avg": 0.15910339, "val/ratio": 0.99038529, "val/ratio_var": 4.043e-05, "val/num_eos_tokens": 98, "lr": 4.18e-06, "episode": 11408, "epoch": 0.57616162, "global_step/max_steps": "713/1238", "percentage": "57.59%", "elapsed_time": "8h 45m 25s", "remaining_time": "6h 26m 52s"} +{"eps": 0, "objective/kl": 42.17440414, "objective/entropy": -193.77468872, "objective/non_score_reward": -2.1087203, "objective/rlhf_reward": 12.28190517, "objective/scores": 14.375, "policy/approxkl_avg": 0.00082981, "policy/clipfrac_avg": 0.00635811, "loss/policy_avg": -0.30597863, "loss/value_avg": 0.67953932, "val/clipfrac_avg": 0.00641429, "policy/entropy_avg": 0.15823698, "val/ratio": 0.98929751, "val/ratio_var": 6.229e-05, "val/num_eos_tokens": 1666, "lr": 4.16e-06, "episode": 11424, "epoch": 0.5769697, "global_step/max_steps": "714/1238", "percentage": "57.67%", "elapsed_time": "8h 47m 9s", "remaining_time": "6h 26m 52s"} +{"eps": 0, "objective/kl": 44.30834579, "objective/entropy": -22.63348389, "objective/non_score_reward": -2.21541739, "objective/rlhf_reward": 14.82364559, "objective/scores": 17.0, "policy/approxkl_avg": 0.00125462, "policy/clipfrac_avg": 0.00773541, "loss/policy_avg": -0.02325748, "loss/value_avg": 0.18199322, "val/clipfrac_avg": 0.00284736, "policy/entropy_avg": 0.24093628, "val/ratio": 0.98407769, "val/ratio_var": 5.949e-05, "val/num_eos_tokens": 302, "lr": 4.15e-06, "episode": 11440, "epoch": 0.57777778, "global_step/max_steps": "715/1238", "percentage": "57.75%", "elapsed_time": "8h 47m 44s", "remaining_time": "6h 26m 1s"} +{"eps": 0, "objective/kl": 40.63113785, "objective/entropy": -168.47468567, "objective/non_score_reward": -2.03155708, "objective/rlhf_reward": 13.07000542, "objective/scores": 15.125, "policy/approxkl_avg": 0.00079342, "policy/clipfrac_avg": 0.0120509, "loss/policy_avg": 0.32375759, "loss/value_avg": 0.38197559, "val/clipfrac_avg": 0.0932284, "policy/entropy_avg": 0.14413071, "val/ratio": 0.99000478, "val/ratio_var": 5.29e-05, "val/num_eos_tokens": 1174, "lr": 4.14e-06, "episode": 11456, "epoch": 0.57858586, "global_step/max_steps": "716/1238", "percentage": "57.84%", "elapsed_time": "8h 49m 10s", "remaining_time": "6h 25m 47s"} +{"eps": 0, "objective/kl": 42.17526627, "objective/entropy": -96.77055359, "objective/non_score_reward": -2.10876322, "objective/rlhf_reward": 14.50061131, "objective/scores": 16.625, "policy/approxkl_avg": 0.00081562, "policy/clipfrac_avg": 0.00808005, "loss/policy_avg": 0.23654325, "loss/value_avg": 1.16687965, "val/clipfrac_avg": 0.02973807, "policy/entropy_avg": 0.14806271, "val/ratio": 0.98993975, "val/ratio_var": 3.871e-05, "val/num_eos_tokens": 297, "lr": 4.12e-06, "episode": 11472, "epoch": 0.57939394, "global_step/max_steps": "717/1238", "percentage": "57.92%", "elapsed_time": "8h 49m 57s", "remaining_time": "6h 25m 5s"} +{"eps": 0, "objective/kl": 42.64197922, "objective/entropy": -58.20504761, "objective/non_score_reward": -2.13209915, "objective/rlhf_reward": 12.20383835, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00095668, "policy/clipfrac_avg": 0.00252684, "loss/policy_avg": -0.11968575, "loss/value_avg": 0.94265896, "val/clipfrac_avg": 0.10988241, "policy/entropy_avg": 0.18893003, "val/ratio": 0.98692143, "val/ratio_var": 8.384e-05, "val/num_eos_tokens": 16, "lr": 4.11e-06, "episode": 11488, "epoch": 0.58020202, "global_step/max_steps": "718/1238", "percentage": "58.00%", "elapsed_time": "8h 50m 41s", "remaining_time": "6h 24m 20s"} +{"eps": 0, "objective/kl": 40.70527649, "objective/entropy": -105.71292114, "objective/non_score_reward": -2.03526378, "objective/rlhf_reward": 13.60536098, "objective/scores": 15.625, "policy/approxkl_avg": 0.00081326, "policy/clipfrac_avg": 0.00411759, "loss/policy_avg": 0.02083624, "loss/value_avg": 0.84992027, "val/clipfrac_avg": 0.04368842, "policy/entropy_avg": 0.15980244, "val/ratio": 0.98956764, "val/ratio_var": 6.342e-05, "val/num_eos_tokens": 428, "lr": 4.1e-06, "episode": 11504, "epoch": 0.5810101, "global_step/max_steps": "719/1238", "percentage": "58.08%", "elapsed_time": "8h 51m 23s", "remaining_time": "6h 23m 34s"} +{"eps": 0, "objective/kl": 32.93721008, "objective/entropy": -100.23733521, "objective/non_score_reward": -1.64686048, "objective/rlhf_reward": 15.13438988, "objective/scores": 16.75, "policy/approxkl_avg": 0.00083853, "policy/clipfrac_avg": 0.00918249, "loss/policy_avg": -0.11627453, "loss/value_avg": 0.73681688, "val/clipfrac_avg": 0.06867726, "policy/entropy_avg": 0.17890549, "val/ratio": 0.98967797, "val/ratio_var": 7.193e-05, "val/num_eos_tokens": 144, "lr": 4.08e-06, "episode": 11520, "epoch": 0.58181818, "global_step/max_steps": "720/1238", "percentage": "58.16%", "elapsed_time": "8h 51m 58s", "remaining_time": "6h 22m 43s"} +{"eps": 0, "objective/kl": 39.98532104, "objective/entropy": -103.39853668, "objective/non_score_reward": -1.99926591, "objective/rlhf_reward": 14.54760933, "objective/scores": 16.5, "policy/approxkl_avg": 0.00088162, "policy/clipfrac_avg": 0.00433416, "loss/policy_avg": -0.06581736, "loss/value_avg": 0.79331005, "val/clipfrac_avg": 0.04114583, "policy/entropy_avg": 0.18559265, "val/ratio": 0.98944968, "val/ratio_var": 6.216e-05, "val/num_eos_tokens": 161, "lr": 4.07e-06, "episode": 11536, "epoch": 0.58262626, "global_step/max_steps": "721/1238", "percentage": "58.24%", "elapsed_time": "8h 52m 32s", "remaining_time": "6h 21m 51s"} +{"eps": 0, "objective/kl": 36.42585754, "objective/entropy": -119.770401, "objective/non_score_reward": -1.821293, "objective/rlhf_reward": 13.46776962, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00080702, "policy/clipfrac_avg": 0.00702257, "loss/policy_avg": 0.44041353, "loss/value_avg": 0.71987891, "val/clipfrac_avg": 0.05163313, "policy/entropy_avg": 0.15318525, "val/ratio": 0.98998153, "val/ratio_var": 4.889e-05, "val/num_eos_tokens": 304, "lr": 4.06e-06, "episode": 11552, "epoch": 0.58343434, "global_step/max_steps": "722/1238", "percentage": "58.32%", "elapsed_time": "8h 53m 10s", "remaining_time": "6h 21m 2s"} +{"eps": 0, "objective/kl": 39.11303711, "objective/entropy": -50.46599579, "objective/non_score_reward": -1.95565188, "objective/rlhf_reward": 13.91153526, "objective/scores": 15.875, "policy/approxkl_avg": 0.00103667, "policy/clipfrac_avg": 0.00608434, "loss/policy_avg": 0.4866091, "loss/value_avg": 1.19530678, "val/clipfrac_avg": 0.09489051, "policy/entropy_avg": 0.20149446, "val/ratio": 0.98675489, "val/ratio_var": 6.938e-05, "val/num_eos_tokens": 36, "lr": 4.04e-06, "episode": 11568, "epoch": 0.58424242, "global_step/max_steps": "723/1238", "percentage": "58.40%", "elapsed_time": "8h 53m 40s", "remaining_time": "6h 20m 8s"} +{"eps": 0, "objective/kl": 37.35525513, "objective/entropy": -112.03081512, "objective/non_score_reward": -1.86776292, "objective/rlhf_reward": 13.49161148, "objective/scores": 15.375, "policy/approxkl_avg": 0.00103555, "policy/clipfrac_avg": 0.00686514, "loss/policy_avg": 0.15895443, "loss/value_avg": 0.56817096, "val/clipfrac_avg": 0.05233292, "policy/entropy_avg": 0.20564032, "val/ratio": 0.98598468, "val/ratio_var": 0.00010151, "val/num_eos_tokens": 347, "lr": 4.03e-06, "episode": 11584, "epoch": 0.58505051, "global_step/max_steps": "724/1238", "percentage": "58.48%", "elapsed_time": "8h 54m 28s", "remaining_time": "6h 19m 26s"} +{"eps": 0, "objective/kl": 41.90481949, "objective/entropy": -69.79931641, "objective/non_score_reward": -2.09524107, "objective/rlhf_reward": 14.85007095, "objective/scores": 17.0, "policy/approxkl_avg": 0.001036, "policy/clipfrac_avg": 0.00456099, "loss/policy_avg": -0.19635428, "loss/value_avg": 0.3302379, "val/clipfrac_avg": 0.0837549, "policy/entropy_avg": 0.20290184, "val/ratio": 0.98649001, "val/ratio_var": 6.111e-05, "val/num_eos_tokens": 261, "lr": 4.02e-06, "episode": 11600, "epoch": 0.58585859, "global_step/max_steps": "725/1238", "percentage": "58.56%", "elapsed_time": "8h 55m 4s", "remaining_time": "6h 18m 36s"} +{"eps": 0, "objective/kl": 32.01334763, "objective/entropy": -104.75285339, "objective/non_score_reward": -1.60066748, "objective/rlhf_reward": 12.7665205, "objective/scores": 14.375, "policy/approxkl_avg": 0.0010035, "policy/clipfrac_avg": 0.0073227, "loss/policy_avg": 0.09991696, "loss/value_avg": 1.06254578, "val/clipfrac_avg": 0.14562753, "policy/entropy_avg": 0.19988441, "val/ratio": 0.98873055, "val/ratio_var": 7.559e-05, "val/num_eos_tokens": 33, "lr": 4e-06, "episode": 11616, "epoch": 0.58666667, "global_step/max_steps": "726/1238", "percentage": "58.64%", "elapsed_time": "8h 56m 0s", "remaining_time": "6h 18m 0s"} +{"eps": 0, "objective/kl": 43.23419952, "objective/entropy": -53.65510941, "objective/non_score_reward": -2.16171026, "objective/rlhf_reward": 12.98672771, "objective/scores": 15.125, "policy/approxkl_avg": 0.00101058, "policy/clipfrac_avg": 0.00668044, "loss/policy_avg": 0.0044315, "loss/value_avg": 0.56515694, "val/clipfrac_avg": 0.0286386, "policy/entropy_avg": 0.17978287, "val/ratio": 0.98807657, "val/ratio_var": 3.508e-05, "val/num_eos_tokens": 25, "lr": 3.99e-06, "episode": 11632, "epoch": 0.58747475, "global_step/max_steps": "727/1238", "percentage": "58.72%", "elapsed_time": "8h 56m 44s", "remaining_time": "6h 17m 16s"} +{"eps": 0, "objective/kl": 40.00040436, "objective/entropy": -147.60580444, "objective/non_score_reward": -2.00002027, "objective/rlhf_reward": 13.30466652, "objective/scores": 15.3125, "policy/approxkl_avg": 0.000822, "policy/clipfrac_avg": 0.01055942, "loss/policy_avg": -0.02680493, "loss/value_avg": 0.32433254, "val/clipfrac_avg": 0.04705826, "policy/entropy_avg": 0.15192938, "val/ratio": 0.98902673, "val/ratio_var": 7.16e-05, "val/num_eos_tokens": 93, "lr": 3.98e-06, "episode": 11648, "epoch": 0.58828283, "global_step/max_steps": "728/1238", "percentage": "58.80%", "elapsed_time": "8h 57m 37s", "remaining_time": "6h 16m 38s"} +{"eps": 0, "objective/kl": 34.32489014, "objective/entropy": -75.26506042, "objective/non_score_reward": -1.71624458, "objective/rlhf_reward": 13.9556303, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008754, "policy/clipfrac_avg": 0.00625351, "loss/policy_avg": 0.06170288, "loss/value_avg": 0.25130609, "val/clipfrac_avg": 0.03564208, "policy/entropy_avg": 0.17108154, "val/ratio": 0.98833925, "val/ratio_var": 5.44e-05, "val/num_eos_tokens": 194, "lr": 3.97e-06, "episode": 11664, "epoch": 0.58909091, "global_step/max_steps": "729/1238", "percentage": "58.89%", "elapsed_time": "8h 58m 12s", "remaining_time": "6h 15m 47s"} +{"eps": 0, "objective/kl": 38.28752136, "objective/entropy": -211.54626465, "objective/non_score_reward": -1.91437602, "objective/rlhf_reward": 13.06218719, "objective/scores": 15.0, "policy/approxkl_avg": 0.00082002, "policy/clipfrac_avg": 0.00927242, "loss/policy_avg": 0.51184291, "loss/value_avg": 0.38098657, "val/clipfrac_avg": 0.13058165, "policy/entropy_avg": 0.15618229, "val/ratio": 0.98929286, "val/ratio_var": 5.911e-05, "val/num_eos_tokens": 191, "lr": 3.95e-06, "episode": 11680, "epoch": 0.58989899, "global_step/max_steps": "730/1238", "percentage": "58.97%", "elapsed_time": "8h 59m 51s", "remaining_time": "6h 15m 41s"} +{"eps": 0, "objective/kl": 33.53625107, "objective/entropy": -155.9753418, "objective/non_score_reward": -1.67681265, "objective/rlhf_reward": 14.19818687, "objective/scores": 15.875, "policy/approxkl_avg": 0.00063739, "policy/clipfrac_avg": 0.01154704, "loss/policy_avg": -0.57558328, "loss/value_avg": 0.65880114, "val/clipfrac_avg": 0.13096668, "policy/entropy_avg": 0.15375042, "val/ratio": 0.99128997, "val/ratio_var": 7.494e-05, "val/num_eos_tokens": 393, "lr": 3.94e-06, "episode": 11696, "epoch": 0.59070707, "global_step/max_steps": "731/1238", "percentage": "59.05%", "elapsed_time": "9h 0m 32s", "remaining_time": "6h 14m 54s"} +{"eps": 0, "objective/kl": 38.77140045, "objective/entropy": -46.36691284, "objective/non_score_reward": -1.93857002, "objective/rlhf_reward": 14.63174248, "objective/scores": 16.625, "policy/approxkl_avg": 0.00102768, "policy/clipfrac_avg": 0.00441912, "loss/policy_avg": 0.08975644, "loss/value_avg": 0.43191338, "val/clipfrac_avg": 0.02601847, "policy/entropy_avg": 0.19952774, "val/ratio": 0.98569298, "val/ratio_var": 5.217e-05, "val/num_eos_tokens": 121, "lr": 3.93e-06, "episode": 11712, "epoch": 0.59151515, "global_step/max_steps": "732/1238", "percentage": "59.13%", "elapsed_time": "9h 1m 5s", "remaining_time": "6h 14m 1s"} +{"eps": 0, "objective/kl": 34.64438629, "objective/entropy": -30.65240479, "objective/non_score_reward": -1.73221946, "objective/rlhf_reward": 16.1271553, "objective/scores": 17.875, "policy/approxkl_avg": 0.00092297, "policy/clipfrac_avg": 0.00531365, "loss/policy_avg": -0.56716013, "loss/value_avg": 1.40099943, "val/clipfrac_avg": 0.16779304, "policy/entropy_avg": 0.16845655, "val/ratio": 0.98807693, "val/ratio_var": 4.936e-05, "val/num_eos_tokens": 257, "lr": 3.91e-06, "episode": 11728, "epoch": 0.59232323, "global_step/max_steps": "733/1238", "percentage": "59.21%", "elapsed_time": "9h 1m 39s", "remaining_time": "6h 13m 10s"} +{"eps": 0, "objective/kl": 38.92997742, "objective/entropy": -115.83007812, "objective/non_score_reward": -1.94649887, "objective/rlhf_reward": 12.38162613, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00069951, "policy/clipfrac_avg": 0.00679882, "loss/policy_avg": 0.1117314, "loss/value_avg": 0.60272133, "val/clipfrac_avg": 0.11902366, "policy/entropy_avg": 0.12786293, "val/ratio": 0.99132067, "val/ratio_var": 3.536e-05, "val/num_eos_tokens": 256, "lr": 3.9e-06, "episode": 11744, "epoch": 0.59313131, "global_step/max_steps": "734/1238", "percentage": "59.29%", "elapsed_time": "9h 2m 25s", "remaining_time": "6h 12m 27s"} +{"eps": 0, "objective/kl": 34.792202, "objective/entropy": -77.05795288, "objective/non_score_reward": -1.7396102, "objective/rlhf_reward": 12.51039028, "objective/scores": 14.25, "policy/approxkl_avg": 0.00090199, "policy/clipfrac_avg": 0.00438591, "loss/policy_avg": 0.07707646, "loss/value_avg": 0.85571885, "val/clipfrac_avg": 0.20910227, "policy/entropy_avg": 0.18432999, "val/ratio": 0.98797095, "val/ratio_var": 7.313e-05, "val/num_eos_tokens": 339, "lr": 3.89e-06, "episode": 11760, "epoch": 0.59393939, "global_step/max_steps": "735/1238", "percentage": "59.37%", "elapsed_time": "9h 3m 1s", "remaining_time": "6h 11m 37s"} +{"eps": 0, "objective/kl": 40.49102402, "objective/entropy": -189.58908081, "objective/non_score_reward": -2.02455139, "objective/rlhf_reward": 12.60826111, "objective/scores": 14.625, "policy/approxkl_avg": 0.0010325, "policy/clipfrac_avg": 0.00882126, "loss/policy_avg": 0.00574364, "loss/value_avg": 0.48230007, "val/clipfrac_avg": 0.03615585, "policy/entropy_avg": 0.18647861, "val/ratio": 0.98676801, "val/ratio_var": 0.00012204, "val/num_eos_tokens": 146, "lr": 3.87e-06, "episode": 11776, "epoch": 0.59474747, "global_step/max_steps": "736/1238", "percentage": "59.45%", "elapsed_time": "9h 4m 46s", "remaining_time": "6h 11m 33s"} +{"eps": 0, "objective/kl": 47.26819992, "objective/entropy": -53.57340622, "objective/non_score_reward": -2.36341, "objective/rlhf_reward": 11.23815346, "objective/scores": 13.625, "policy/approxkl_avg": 0.00104169, "policy/clipfrac_avg": 0.00632802, "loss/policy_avg": 0.11549819, "loss/value_avg": 0.41911316, "val/clipfrac_avg": 0.0545404, "policy/entropy_avg": 0.21573257, "val/ratio": 0.98635179, "val/ratio_var": 5.174e-05, "val/num_eos_tokens": 145, "lr": 3.86e-06, "episode": 11792, "epoch": 0.59555556, "global_step/max_steps": "737/1238", "percentage": "59.53%", "elapsed_time": "9h 5m 21s", "remaining_time": "6h 10m 43s"} +{"eps": 0, "objective/kl": 41.81903839, "objective/entropy": -9.55144501, "objective/non_score_reward": -2.09095192, "objective/rlhf_reward": 15.36217308, "objective/scores": 17.5, "policy/approxkl_avg": 0.0010775, "policy/clipfrac_avg": 0.00724438, "loss/policy_avg": 0.09252296, "loss/value_avg": 0.24368465, "val/clipfrac_avg": 0.04672763, "policy/entropy_avg": 0.20846558, "val/ratio": 0.98597085, "val/ratio_var": 4.752e-05, "val/num_eos_tokens": 70, "lr": 3.85e-06, "episode": 11808, "epoch": 0.59636364, "global_step/max_steps": "738/1238", "percentage": "59.61%", "elapsed_time": "9h 5m 50s", "remaining_time": "6h 9m 48s"} +{"eps": 0, "objective/kl": 32.57723999, "objective/entropy": -109.43662262, "objective/non_score_reward": -1.62886214, "objective/rlhf_reward": 13.33207512, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00093117, "policy/clipfrac_avg": 0.00774637, "loss/policy_avg": -0.47144565, "loss/value_avg": 0.96598256, "val/clipfrac_avg": 0.15425864, "policy/entropy_avg": 0.20248222, "val/ratio": 0.9888984, "val/ratio_var": 8.098e-05, "val/num_eos_tokens": 140, "lr": 3.84e-06, "episode": 11824, "epoch": 0.59717172, "global_step/max_steps": "739/1238", "percentage": "59.69%", "elapsed_time": "9h 6m 21s", "remaining_time": "6h 8m 55s"} +{"eps": 0, "objective/kl": 38.61618042, "objective/entropy": -82.36312103, "objective/non_score_reward": -1.9308089, "objective/rlhf_reward": 12.72544098, "objective/scores": 14.625, "policy/approxkl_avg": 0.00089917, "policy/clipfrac_avg": 0.01843995, "loss/policy_avg": 0.39348739, "loss/value_avg": 0.51605165, "val/clipfrac_avg": 0.00676593, "policy/entropy_avg": 0.22455025, "val/ratio": 0.98834866, "val/ratio_var": 5.282e-05, "val/num_eos_tokens": 30, "lr": 3.82e-06, "episode": 11840, "epoch": 0.5979798, "global_step/max_steps": "740/1238", "percentage": "59.77%", "elapsed_time": "9h 7m 23s", "remaining_time": "6h 8m 23s"} +{"eps": 0, "objective/kl": 46.57951736, "objective/entropy": -89.22055054, "objective/non_score_reward": -2.32897592, "objective/rlhf_reward": 13.48352432, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00099609, "policy/clipfrac_avg": 0.00589654, "loss/policy_avg": -0.06747609, "loss/value_avg": 0.31143257, "val/clipfrac_avg": 0.06993684, "policy/entropy_avg": 0.19659424, "val/ratio": 0.98753142, "val/ratio_var": 4.793e-05, "val/num_eos_tokens": 107, "lr": 3.81e-06, "episode": 11856, "epoch": 0.59878788, "global_step/max_steps": "741/1238", "percentage": "59.85%", "elapsed_time": "9h 8m 32s", "remaining_time": "6h 7m 54s"} +{"eps": 0, "objective/kl": 36.62680054, "objective/entropy": -104.6260376, "objective/non_score_reward": -1.83133996, "objective/rlhf_reward": 14.77803421, "objective/scores": 16.625, "policy/approxkl_avg": 0.00087605, "policy/clipfrac_avg": 0.00524907, "loss/policy_avg": -0.27379957, "loss/value_avg": 0.47723848, "val/clipfrac_avg": 0.06328598, "policy/entropy_avg": 0.17531395, "val/ratio": 0.98858213, "val/ratio_var": 5.911e-05, "val/num_eos_tokens": 91, "lr": 3.8e-06, "episode": 11872, "epoch": 0.59959596, "global_step/max_steps": "742/1238", "percentage": "59.94%", "elapsed_time": "9h 9m 25s", "remaining_time": "6h 7m 16s"} +{"eps": 0, "objective/kl": 35.78708649, "objective/entropy": -123.75762177, "objective/non_score_reward": -1.78935444, "objective/rlhf_reward": 14.86689568, "objective/scores": 16.625, "policy/approxkl_avg": 0.000777, "policy/clipfrac_avg": 0.00561767, "loss/policy_avg": -0.11605306, "loss/value_avg": 0.30427811, "val/clipfrac_avg": 0.04839505, "policy/entropy_avg": 0.15231323, "val/ratio": 0.99056536, "val/ratio_var": 5.71e-05, "val/num_eos_tokens": 386, "lr": 3.78e-06, "episode": 11888, "epoch": 0.60040404, "global_step/max_steps": "743/1238", "percentage": "60.02%", "elapsed_time": "9h 10m 5s", "remaining_time": "6h 6m 28s"} +{"eps": 0, "objective/kl": 43.74962616, "objective/entropy": -39.34166336, "objective/non_score_reward": -2.1874814, "objective/rlhf_reward": 12.64845657, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00108957, "policy/clipfrac_avg": 0.00520081, "loss/policy_avg": -0.02294029, "loss/value_avg": 0.72536689, "val/clipfrac_avg": 0.01829146, "policy/entropy_avg": 0.22774124, "val/ratio": 0.98533857, "val/ratio_var": 6.193e-05, "val/num_eos_tokens": 71, "lr": 3.77e-06, "episode": 11904, "epoch": 0.60121212, "global_step/max_steps": "744/1238", "percentage": "60.10%", "elapsed_time": "9h 10m 53s", "remaining_time": "6h 5m 46s"} +{"eps": 0, "objective/kl": 33.78061676, "objective/entropy": -79.38550568, "objective/non_score_reward": -1.68903089, "objective/rlhf_reward": 14.17815685, "objective/scores": 15.875, "policy/approxkl_avg": 0.00086614, "policy/clipfrac_avg": 0.00355887, "loss/policy_avg": 0.13283888, "loss/value_avg": 1.33907461, "val/clipfrac_avg": 0.05471283, "policy/entropy_avg": 0.15964603, "val/ratio": 0.98979342, "val/ratio_var": 8.216e-05, "val/num_eos_tokens": 81, "lr": 3.76e-06, "episode": 11920, "epoch": 0.6020202, "global_step/max_steps": "745/1238", "percentage": "60.18%", "elapsed_time": "9h 11m 24s", "remaining_time": "6h 4m 53s"} +{"eps": 0, "objective/kl": 42.90866089, "objective/entropy": -124.25365448, "objective/non_score_reward": -2.14543295, "objective/rlhf_reward": 11.87019157, "objective/scores": 14.0, "policy/approxkl_avg": 0.00094702, "policy/clipfrac_avg": 0.00687884, "loss/policy_avg": -0.29737264, "loss/value_avg": 0.63142705, "val/clipfrac_avg": 0.07988048, "policy/entropy_avg": 0.18181038, "val/ratio": 0.98801887, "val/ratio_var": 8.262e-05, "val/num_eos_tokens": 329, "lr": 3.74e-06, "episode": 11936, "epoch": 0.60282828, "global_step/max_steps": "746/1238", "percentage": "60.26%", "elapsed_time": "9h 12m 2s", "remaining_time": "6h 4m 4s"} +{"eps": 0, "objective/kl": 43.51027679, "objective/entropy": -72.31340027, "objective/non_score_reward": -2.17551398, "objective/rlhf_reward": 12.34792328, "objective/scores": 14.5, "policy/approxkl_avg": 0.00099414, "policy/clipfrac_avg": 0.00658875, "loss/policy_avg": 0.1245454, "loss/value_avg": 0.44872367, "val/clipfrac_avg": 0.15402031, "policy/entropy_avg": 0.16569901, "val/ratio": 0.98887908, "val/ratio_var": 2.404e-05, "val/num_eos_tokens": 77, "lr": 3.73e-06, "episode": 11952, "epoch": 0.60363636, "global_step/max_steps": "747/1238", "percentage": "60.34%", "elapsed_time": "9h 12m 42s", "remaining_time": "6h 3m 17s"} +{"eps": 0, "objective/kl": 35.28123474, "objective/entropy": -103.352005, "objective/non_score_reward": -1.76406169, "objective/rlhf_reward": 13.27500057, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00081918, "policy/clipfrac_avg": 0.00634311, "loss/policy_avg": -0.15478614, "loss/value_avg": 0.4902263, "val/clipfrac_avg": 0.04430588, "policy/entropy_avg": 0.14904118, "val/ratio": 0.99087894, "val/ratio_var": 4.545e-05, "val/num_eos_tokens": 124, "lr": 3.72e-06, "episode": 11968, "epoch": 0.60444444, "global_step/max_steps": "748/1238", "percentage": "60.42%", "elapsed_time": "9h 13m 30s", "remaining_time": "6h 2m 35s"} +{"eps": 0, "objective/kl": 36.7634964, "objective/entropy": -111.66493225, "objective/non_score_reward": -1.83817482, "objective/rlhf_reward": 13.69307518, "objective/scores": 15.5, "policy/approxkl_avg": 0.00089186, "policy/clipfrac_avg": 0.01091066, "loss/policy_avg": 0.11601712, "loss/value_avg": 0.61755562, "val/clipfrac_avg": 0.06755827, "policy/entropy_avg": 0.16454506, "val/ratio": 0.98865008, "val/ratio_var": 6.256e-05, "val/num_eos_tokens": 358, "lr": 3.71e-06, "episode": 11984, "epoch": 0.60525253, "global_step/max_steps": "749/1238", "percentage": "60.50%", "elapsed_time": "9h 14m 6s", "remaining_time": "6h 1m 45s"} +{"eps": 0, "objective/kl": 32.27269745, "objective/entropy": -63.29137039, "objective/non_score_reward": -1.61363494, "objective/rlhf_reward": 14.64417744, "objective/scores": 16.25, "policy/approxkl_avg": 0.00094903, "policy/clipfrac_avg": 0.00682455, "loss/policy_avg": -0.04627025, "loss/value_avg": 0.62246895, "val/clipfrac_avg": 0.18630268, "policy/entropy_avg": 0.17229939, "val/ratio": 0.98780209, "val/ratio_var": 5.455e-05, "val/num_eos_tokens": 11, "lr": 3.69e-06, "episode": 12000, "epoch": 0.60606061, "global_step/max_steps": "750/1238", "percentage": "60.58%", "elapsed_time": "9h 14m 48s", "remaining_time": "6h 0m 59s"} +{"eps": 0, "objective/kl": 34.44614029, "objective/entropy": -54.77946472, "objective/non_score_reward": -1.72230697, "objective/rlhf_reward": 14.89488029, "objective/scores": 16.625, "policy/approxkl_avg": 0.00117799, "policy/clipfrac_avg": 0.02440253, "loss/policy_avg": -0.17890789, "loss/value_avg": 0.71409476, "val/clipfrac_avg": 0.04182716, "policy/entropy_avg": 0.20982552, "val/ratio": 0.98549396, "val/ratio_var": 0.00014134, "val/num_eos_tokens": 42, "lr": 3.68e-06, "episode": 12016, "epoch": 0.60686869, "global_step/max_steps": "751/1238", "percentage": "60.66%", "elapsed_time": "9h 15m 21s", "remaining_time": "6h 0m 7s"} +{"eps": 0, "objective/kl": 37.6799469, "objective/entropy": -70.77661133, "objective/non_score_reward": -1.88399744, "objective/rlhf_reward": 12.10819054, "objective/scores": 14.0, "policy/approxkl_avg": 0.00093735, "policy/clipfrac_avg": 0.00521011, "loss/policy_avg": -0.00150077, "loss/value_avg": 0.25441027, "val/clipfrac_avg": 0.02112862, "policy/entropy_avg": 0.18710041, "val/ratio": 0.98740619, "val/ratio_var": 5.213e-05, "val/num_eos_tokens": 218, "lr": 3.67e-06, "episode": 12032, "epoch": 0.60767677, "global_step/max_steps": "752/1238", "percentage": "60.74%", "elapsed_time": "9h 15m 54s", "remaining_time": "5h 59m 15s"} +{"eps": 0, "objective/kl": 44.55897141, "objective/entropy": -89.17417908, "objective/non_score_reward": -2.22794867, "objective/rlhf_reward": 12.17830086, "objective/scores": 14.375, "policy/approxkl_avg": 0.00093756, "policy/clipfrac_avg": 0.00588262, "loss/policy_avg": 0.05351485, "loss/value_avg": 0.28849858, "val/clipfrac_avg": 0.02156676, "policy/entropy_avg": 0.18878555, "val/ratio": 0.9880842, "val/ratio_var": 5.728e-05, "val/num_eos_tokens": 120, "lr": 3.65e-06, "episode": 12048, "epoch": 0.60848485, "global_step/max_steps": "753/1238", "percentage": "60.82%", "elapsed_time": "9h 16m 29s", "remaining_time": "5h 58m 25s"} +{"eps": 0, "objective/kl": 43.28697968, "objective/entropy": -114.39792633, "objective/non_score_reward": -2.16434908, "objective/rlhf_reward": 14.2262764, "objective/scores": 16.375, "policy/approxkl_avg": 0.00086613, "policy/clipfrac_avg": 0.00503151, "loss/policy_avg": -0.12424174, "loss/value_avg": 0.35938853, "val/clipfrac_avg": 0.14269608, "policy/entropy_avg": 0.16837502, "val/ratio": 0.98946285, "val/ratio_var": 5.016e-05, "val/num_eos_tokens": 391, "lr": 3.64e-06, "episode": 12064, "epoch": 0.60929293, "global_step/max_steps": "754/1238", "percentage": "60.90%", "elapsed_time": "9h 17m 3s", "remaining_time": "5h 57m 34s"} +{"eps": 0, "objective/kl": 38.83340073, "objective/entropy": -164.57290649, "objective/non_score_reward": -1.94167006, "objective/rlhf_reward": 14.24582958, "objective/scores": 16.25, "policy/approxkl_avg": 0.00081408, "policy/clipfrac_avg": 0.00435155, "loss/policy_avg": -0.28214258, "loss/value_avg": 0.38819343, "val/clipfrac_avg": 0.09525478, "policy/entropy_avg": 0.15729976, "val/ratio": 0.98944622, "val/ratio_var": 5.319e-05, "val/num_eos_tokens": 336, "lr": 3.63e-06, "episode": 12080, "epoch": 0.61010101, "global_step/max_steps": "755/1238", "percentage": "60.99%", "elapsed_time": "9h 17m 58s", "remaining_time": "5h 56m 57s"} +{"eps": 0, "objective/kl": 32.80990219, "objective/entropy": -61.17611694, "objective/non_score_reward": -1.64049506, "objective/rlhf_reward": 16.2423172, "objective/scores": 17.875, "policy/approxkl_avg": 0.00095978, "policy/clipfrac_avg": 0.00619729, "loss/policy_avg": 0.05094696, "loss/value_avg": 0.42604765, "val/clipfrac_avg": 0.01815173, "policy/entropy_avg": 0.16566753, "val/ratio": 0.98940444, "val/ratio_var": 4.711e-05, "val/num_eos_tokens": 163, "lr": 3.62e-06, "episode": 12096, "epoch": 0.61090909, "global_step/max_steps": "756/1238", "percentage": "61.07%", "elapsed_time": "9h 18m 31s", "remaining_time": "5h 56m 6s"} +{"eps": 0, "objective/kl": 34.21899033, "objective/entropy": -52.30070877, "objective/non_score_reward": -1.71094966, "objective/rlhf_reward": 13.2734251, "objective/scores": 15.0, "policy/approxkl_avg": 0.00094502, "policy/clipfrac_avg": 0.00490978, "loss/policy_avg": -0.04046983, "loss/value_avg": 0.37466162, "val/clipfrac_avg": 0.09231512, "policy/entropy_avg": 0.18058777, "val/ratio": 0.98773694, "val/ratio_var": 4.408e-05, "val/num_eos_tokens": 105, "lr": 3.6e-06, "episode": 12112, "epoch": 0.61171717, "global_step/max_steps": "757/1238", "percentage": "61.15%", "elapsed_time": "9h 19m 14s", "remaining_time": "5h 55m 20s"} +{"eps": 0, "objective/kl": 42.89472961, "objective/entropy": -33.08611298, "objective/non_score_reward": -2.14473653, "objective/rlhf_reward": 14.79276371, "objective/scores": 17.0, "policy/approxkl_avg": 0.00116286, "policy/clipfrac_avg": 0.00601825, "loss/policy_avg": -0.07909817, "loss/value_avg": 0.28900239, "val/clipfrac_avg": 0.05997534, "policy/entropy_avg": 0.21850204, "val/ratio": 0.98589987, "val/ratio_var": 4.603e-05, "val/num_eos_tokens": 208, "lr": 3.59e-06, "episode": 12128, "epoch": 0.61252525, "global_step/max_steps": "758/1238", "percentage": "61.23%", "elapsed_time": "9h 19m 51s", "remaining_time": "5h 54m 31s"} +{"eps": 0, "objective/kl": 34.86107635, "objective/entropy": -57.00241852, "objective/non_score_reward": -1.74305367, "objective/rlhf_reward": 12.42100906, "objective/scores": 14.1875, "policy/approxkl_avg": 0.00096963, "policy/clipfrac_avg": 0.00579453, "loss/policy_avg": 0.08223092, "loss/value_avg": 0.452171, "val/clipfrac_avg": 0.03934604, "policy/entropy_avg": 0.18263245, "val/ratio": 0.98803794, "val/ratio_var": 4.821e-05, "val/num_eos_tokens": 137, "lr": 3.58e-06, "episode": 12144, "epoch": 0.61333333, "global_step/max_steps": "759/1238", "percentage": "61.31%", "elapsed_time": "9h 20m 29s", "remaining_time": "5h 53m 43s"} +{"eps": 0, "objective/kl": 40.15751648, "objective/entropy": -58.77183533, "objective/non_score_reward": -2.00787592, "objective/rlhf_reward": 13.17181206, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00097552, "policy/clipfrac_avg": 0.00756999, "loss/policy_avg": 0.01290392, "loss/value_avg": 0.22128555, "val/clipfrac_avg": 0.00769504, "policy/entropy_avg": 0.17936611, "val/ratio": 0.98791075, "val/ratio_var": 9.123e-05, "val/num_eos_tokens": 194, "lr": 3.56e-06, "episode": 12160, "epoch": 0.61414141, "global_step/max_steps": "760/1238", "percentage": "61.39%", "elapsed_time": "9h 21m 3s", "remaining_time": "5h 52m 52s"} +{"eps": 0, "objective/kl": 36.40907288, "objective/entropy": -92.11294556, "objective/non_score_reward": -1.82045388, "objective/rlhf_reward": 13.46860886, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00099491, "policy/clipfrac_avg": 0.0120312, "loss/policy_avg": 0.23269692, "loss/value_avg": 0.6034168, "val/clipfrac_avg": 0.03142272, "policy/entropy_avg": 0.19497204, "val/ratio": 0.98710519, "val/ratio_var": 7.474e-05, "val/num_eos_tokens": 210, "lr": 3.55e-06, "episode": 12176, "epoch": 0.61494949, "global_step/max_steps": "761/1238", "percentage": "61.47%", "elapsed_time": "9h 21m 48s", "remaining_time": "5h 52m 8s"} +{"eps": 0, "objective/kl": 32.94031525, "objective/entropy": -42.83490753, "objective/non_score_reward": -1.64701581, "objective/rlhf_reward": 13.56782818, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00115914, "policy/clipfrac_avg": 0.00448435, "loss/policy_avg": -0.01448702, "loss/value_avg": 1.0809375, "val/clipfrac_avg": 0.10175604, "policy/entropy_avg": 0.21469688, "val/ratio": 0.9864502, "val/ratio_var": 8.948e-05, "val/num_eos_tokens": 453, "lr": 3.54e-06, "episode": 12192, "epoch": 0.61575758, "global_step/max_steps": "762/1238", "percentage": "61.55%", "elapsed_time": "9h 22m 20s", "remaining_time": "5h 51m 16s"} +{"eps": 0, "objective/kl": 31.28769302, "objective/entropy": -71.51974487, "objective/non_score_reward": -1.5643847, "objective/rlhf_reward": 15.21686554, "objective/scores": 16.75, "policy/approxkl_avg": 0.00078111, "policy/clipfrac_avg": 0.00535214, "loss/policy_avg": -0.04109918, "loss/value_avg": 1.30143785, "val/clipfrac_avg": 0.05970404, "policy/entropy_avg": 0.1676451, "val/ratio": 0.98945665, "val/ratio_var": 6.841e-05, "val/num_eos_tokens": 203, "lr": 3.53e-06, "episode": 12208, "epoch": 0.61656566, "global_step/max_steps": "763/1238", "percentage": "61.63%", "elapsed_time": "9h 22m 51s", "remaining_time": "5h 50m 23s"} +{"eps": 0, "objective/kl": 41.09782028, "objective/entropy": -99.98547363, "objective/non_score_reward": -2.05489087, "objective/rlhf_reward": 10.62479591, "objective/scores": 12.6875, "policy/approxkl_avg": 0.0009035, "policy/clipfrac_avg": 0.01042742, "loss/policy_avg": 0.11212642, "loss/value_avg": 0.72555435, "val/clipfrac_avg": 0.12764864, "policy/entropy_avg": 0.17998505, "val/ratio": 0.98847735, "val/ratio_var": 8.91e-05, "val/num_eos_tokens": 226, "lr": 3.51e-06, "episode": 12224, "epoch": 0.61737374, "global_step/max_steps": "764/1238", "percentage": "61.71%", "elapsed_time": "9h 23m 37s", "remaining_time": "5h 49m 40s"} +{"eps": 0, "objective/kl": 41.46658325, "objective/entropy": -135.23390198, "objective/non_score_reward": -2.07332921, "objective/rlhf_reward": 12.94229507, "objective/scores": 15.0, "policy/approxkl_avg": 0.0009621, "policy/clipfrac_avg": 0.00628145, "loss/policy_avg": 0.3775883, "loss/value_avg": 0.74658, "val/clipfrac_avg": 0.10096653, "policy/entropy_avg": 0.17942047, "val/ratio": 0.98775685, "val/ratio_var": 7.068e-05, "val/num_eos_tokens": 233, "lr": 3.5e-06, "episode": 12240, "epoch": 0.61818182, "global_step/max_steps": "765/1238", "percentage": "61.79%", "elapsed_time": "9h 24m 43s", "remaining_time": "5h 49m 10s"} +{"eps": 0, "objective/kl": 39.61420441, "objective/entropy": -86.19502258, "objective/non_score_reward": -1.98071003, "objective/rlhf_reward": 13.70678902, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00095164, "policy/clipfrac_avg": 0.01008635, "loss/policy_avg": -0.1078265, "loss/value_avg": 0.49963599, "val/clipfrac_avg": 0.05575939, "policy/entropy_avg": 0.17633438, "val/ratio": 0.98848605, "val/ratio_var": 5.413e-05, "val/num_eos_tokens": 426, "lr": 3.49e-06, "episode": 12256, "epoch": 0.6189899, "global_step/max_steps": "766/1238", "percentage": "61.87%", "elapsed_time": "9h 25m 22s", "remaining_time": "5h 48m 22s"} +{"eps": 0, "objective/kl": 38.48719788, "objective/entropy": -78.26728058, "objective/non_score_reward": -1.92436004, "objective/rlhf_reward": 11.66157722, "objective/scores": 13.5625, "policy/approxkl_avg": 0.00094667, "policy/clipfrac_avg": 0.00427088, "loss/policy_avg": 0.06119356, "loss/value_avg": 0.40072089, "val/clipfrac_avg": 0.11514523, "policy/entropy_avg": 0.17498779, "val/ratio": 0.98746598, "val/ratio_var": 7.443e-05, "val/num_eos_tokens": 415, "lr": 3.48e-06, "episode": 12272, "epoch": 0.61979798, "global_step/max_steps": "767/1238", "percentage": "61.95%", "elapsed_time": "9h 25m 49s", "remaining_time": "5h 47m 28s"} +{"eps": 0, "objective/kl": 41.06260681, "objective/entropy": -45.31693649, "objective/non_score_reward": -2.05313039, "objective/rlhf_reward": 13.8609333, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00116516, "policy/clipfrac_avg": 0.00471823, "loss/policy_avg": -0.07818018, "loss/value_avg": 0.58829373, "val/clipfrac_avg": 0.16338575, "policy/entropy_avg": 0.20906067, "val/ratio": 0.9864977, "val/ratio_var": 8.176e-05, "val/num_eos_tokens": 9, "lr": 3.46e-06, "episode": 12288, "epoch": 0.62060606, "global_step/max_steps": "768/1238", "percentage": "62.04%", "elapsed_time": "9h 26m 26s", "remaining_time": "5h 46m 38s"} +{"eps": 0, "objective/kl": 35.79455185, "objective/entropy": -103.96510315, "objective/non_score_reward": -1.78972769, "objective/rlhf_reward": 13.50714684, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00100841, "policy/clipfrac_avg": 0.00700424, "loss/policy_avg": -0.3265214, "loss/value_avg": 0.58787441, "val/clipfrac_avg": 0.12878293, "policy/entropy_avg": 0.19075871, "val/ratio": 0.98615021, "val/ratio_var": 8.974e-05, "val/num_eos_tokens": 8, "lr": 3.45e-06, "episode": 12304, "epoch": 0.62141414, "global_step/max_steps": "769/1238", "percentage": "62.12%", "elapsed_time": "9h 27m 10s", "remaining_time": "5h 45m 54s"} +{"eps": 0, "objective/kl": 31.87890625, "objective/entropy": -92.59934998, "objective/non_score_reward": -1.59394526, "objective/rlhf_reward": 14.42949295, "objective/scores": 16.0, "policy/approxkl_avg": 0.00078361, "policy/clipfrac_avg": 0.00406106, "loss/policy_avg": -0.24864814, "loss/value_avg": 0.7036674, "val/clipfrac_avg": 0.14356926, "policy/entropy_avg": 0.13613224, "val/ratio": 0.99096233, "val/ratio_var": 6.253e-05, "val/num_eos_tokens": 22, "lr": 3.44e-06, "episode": 12320, "epoch": 0.62222222, "global_step/max_steps": "770/1238", "percentage": "62.20%", "elapsed_time": "9h 27m 44s", "remaining_time": "5h 45m 4s"} +{"eps": 0, "objective/kl": 32.05082321, "objective/entropy": -201.82592773, "objective/non_score_reward": -1.60254121, "objective/rlhf_reward": 13.84277153, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00092248, "policy/clipfrac_avg": 0.00409099, "loss/policy_avg": -0.11106159, "loss/value_avg": 0.77197874, "val/clipfrac_avg": 0.06381858, "policy/entropy_avg": 0.194525, "val/ratio": 0.98732215, "val/ratio_var": 0.000108, "val/num_eos_tokens": 137, "lr": 3.42e-06, "episode": 12336, "epoch": 0.6230303, "global_step/max_steps": "771/1238", "percentage": "62.28%", "elapsed_time": "9h 29m 24s", "remaining_time": "5h 44m 53s"} +{"eps": 0, "objective/kl": 28.94087791, "objective/entropy": -97.80130768, "objective/non_score_reward": -1.4470439, "objective/rlhf_reward": 13.57248688, "objective/scores": 15.0, "policy/approxkl_avg": 0.00082332, "policy/clipfrac_avg": 0.00752334, "loss/policy_avg": -0.31477022, "loss/value_avg": 1.01987398, "val/clipfrac_avg": 0.02168833, "policy/entropy_avg": 0.17452717, "val/ratio": 0.98850316, "val/ratio_var": 6.794e-05, "val/num_eos_tokens": 116, "lr": 3.41e-06, "episode": 12352, "epoch": 0.62383838, "global_step/max_steps": "772/1238", "percentage": "62.36%", "elapsed_time": "9h 30m 0s", "remaining_time": "5h 44m 4s"} +{"eps": 0, "objective/kl": 33.00112915, "objective/entropy": -61.79416656, "objective/non_score_reward": -1.65005648, "objective/rlhf_reward": 12.94369316, "objective/scores": 14.625, "policy/approxkl_avg": 0.00094176, "policy/clipfrac_avg": 0.0052822, "loss/policy_avg": -0.01992, "loss/value_avg": 0.31691867, "val/clipfrac_avg": 0.05137361, "policy/entropy_avg": 0.17323112, "val/ratio": 0.98893523, "val/ratio_var": 3.957e-05, "val/num_eos_tokens": 13, "lr": 3.4e-06, "episode": 12368, "epoch": 0.62464646, "global_step/max_steps": "773/1238", "percentage": "62.44%", "elapsed_time": "9h 30m 26s", "remaining_time": "5h 43m 9s"} +{"eps": 0, "objective/kl": 29.43255615, "objective/entropy": -73.91504669, "objective/non_score_reward": -1.47162783, "objective/rlhf_reward": 13.29399681, "objective/scores": 14.75, "policy/approxkl_avg": 0.00101805, "policy/clipfrac_avg": 0.00508894, "loss/policy_avg": 0.01005184, "loss/value_avg": 0.54140192, "val/clipfrac_avg": 0.04191375, "policy/entropy_avg": 0.21539116, "val/ratio": 0.98631591, "val/ratio_var": 8.102e-05, "val/num_eos_tokens": 214, "lr": 3.39e-06, "episode": 12384, "epoch": 0.62545455, "global_step/max_steps": "774/1238", "percentage": "62.52%", "elapsed_time": "9h 31m 13s", "remaining_time": "5h 42m 26s"} +{"eps": 0, "objective/kl": 31.84616089, "objective/entropy": -75.78406525, "objective/non_score_reward": -1.59230816, "objective/rlhf_reward": 11.94675446, "objective/scores": 13.5625, "policy/approxkl_avg": 0.00090443, "policy/clipfrac_avg": 0.00500277, "loss/policy_avg": -0.04047441, "loss/value_avg": 0.30427632, "val/clipfrac_avg": 0.04153668, "policy/entropy_avg": 0.18775272, "val/ratio": 0.98775291, "val/ratio_var": 5.462e-05, "val/num_eos_tokens": 243, "lr": 3.37e-06, "episode": 12400, "epoch": 0.62626263, "global_step/max_steps": "775/1238", "percentage": "62.60%", "elapsed_time": "9h 31m 54s", "remaining_time": "5h 41m 40s"} +{"eps": 0, "objective/kl": 35.97426605, "objective/entropy": -97.84140015, "objective/non_score_reward": -1.79871321, "objective/rlhf_reward": 13.68566132, "objective/scores": 15.5, "policy/approxkl_avg": 0.00093632, "policy/clipfrac_avg": 0.00350527, "loss/policy_avg": -0.25257227, "loss/value_avg": 0.47636741, "val/clipfrac_avg": 0.07260937, "policy/entropy_avg": 0.18935299, "val/ratio": 0.98775864, "val/ratio_var": 0.00010293, "val/num_eos_tokens": 26, "lr": 3.36e-06, "episode": 12416, "epoch": 0.62707071, "global_step/max_steps": "776/1238", "percentage": "62.68%", "elapsed_time": "9h 32m 26s", "remaining_time": "5h 40m 48s"} +{"eps": 0, "objective/kl": 29.19748688, "objective/entropy": -40.24072266, "objective/non_score_reward": -1.45987439, "objective/rlhf_reward": 12.88387489, "objective/scores": 14.375, "policy/approxkl_avg": 0.00108318, "policy/clipfrac_avg": 0.02700811, "loss/policy_avg": 0.41764987, "loss/value_avg": 0.86290616, "val/clipfrac_avg": 0.08423963, "policy/entropy_avg": 0.16212845, "val/ratio": 0.98857355, "val/ratio_var": 8.724e-05, "val/num_eos_tokens": 192, "lr": 3.35e-06, "episode": 12432, "epoch": 0.62787879, "global_step/max_steps": "777/1238", "percentage": "62.76%", "elapsed_time": "9h 32m 44s", "remaining_time": "5h 39m 48s"} +{"eps": 0, "objective/kl": 33.57006073, "objective/entropy": -69.47827148, "objective/non_score_reward": -1.67850304, "objective/rlhf_reward": 14.05587196, "objective/scores": 15.75, "policy/approxkl_avg": 0.00088992, "policy/clipfrac_avg": 0.00682587, "loss/policy_avg": 0.22420269, "loss/value_avg": 0.9421165, "val/clipfrac_avg": 0.15017873, "policy/entropy_avg": 0.14767838, "val/ratio": 0.99054188, "val/ratio_var": 4.142e-05, "val/num_eos_tokens": 109, "lr": 3.34e-06, "episode": 12448, "epoch": 0.62868687, "global_step/max_steps": "778/1238", "percentage": "62.84%", "elapsed_time": "9h 33m 17s", "remaining_time": "5h 38m 57s"} +{"eps": 0, "objective/kl": 31.9446907, "objective/entropy": -182.4083252, "objective/non_score_reward": -1.59723449, "objective/rlhf_reward": 12.04339027, "objective/scores": 13.625, "policy/approxkl_avg": 0.00087189, "policy/clipfrac_avg": 0.00461358, "loss/policy_avg": 0.03614666, "loss/value_avg": 0.73870218, "val/clipfrac_avg": 0.10708205, "policy/entropy_avg": 0.17332268, "val/ratio": 0.98805195, "val/ratio_var": 9.138e-05, "val/num_eos_tokens": 28, "lr": 3.32e-06, "episode": 12464, "epoch": 0.62949495, "global_step/max_steps": "779/1238", "percentage": "62.92%", "elapsed_time": "9h 34m 57s", "remaining_time": "5h 38m 46s"} +{"eps": 0, "objective/kl": 22.97904396, "objective/entropy": -80.43102264, "objective/non_score_reward": -1.14895225, "objective/rlhf_reward": 13.28073502, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00075399, "policy/clipfrac_avg": 0.01330423, "loss/policy_avg": 0.49124384, "loss/value_avg": 1.74493492, "val/clipfrac_avg": 0.12828538, "policy/entropy_avg": 0.16235483, "val/ratio": 0.99166632, "val/ratio_var": 4.579e-05, "val/num_eos_tokens": 261, "lr": 3.31e-06, "episode": 12480, "epoch": 0.63030303, "global_step/max_steps": "780/1238", "percentage": "63.00%", "elapsed_time": "9h 35m 45s", "remaining_time": "5h 38m 4s"} +{"eps": 0, "objective/kl": 37.96983337, "objective/entropy": -96.58609009, "objective/non_score_reward": -1.89849162, "objective/rlhf_reward": 12.74213314, "objective/scores": 14.625, "policy/approxkl_avg": 0.00089191, "policy/clipfrac_avg": 0.00413879, "loss/policy_avg": -0.02106084, "loss/value_avg": 0.26465464, "val/clipfrac_avg": 0.02277688, "policy/entropy_avg": 0.17605591, "val/ratio": 0.98803139, "val/ratio_var": 5.301e-05, "val/num_eos_tokens": 232, "lr": 3.3e-06, "episode": 12496, "epoch": 0.63111111, "global_step/max_steps": "781/1238", "percentage": "63.09%", "elapsed_time": "9h 36m 19s", "remaining_time": "5h 37m 13s"} +{"eps": 0, "objective/kl": 30.8980217, "objective/entropy": -113.97618103, "objective/non_score_reward": -1.54490113, "objective/rlhf_reward": 15.89259911, "objective/scores": 17.5, "policy/approxkl_avg": 0.00081224, "policy/clipfrac_avg": 0.00718042, "loss/policy_avg": -0.21595685, "loss/value_avg": 0.16683111, "val/clipfrac_avg": 0.0471012, "policy/entropy_avg": 0.15228558, "val/ratio": 0.99051201, "val/ratio_var": 4.686e-05, "val/num_eos_tokens": 88, "lr": 3.29e-06, "episode": 12512, "epoch": 0.63191919, "global_step/max_steps": "782/1238", "percentage": "63.17%", "elapsed_time": "9h 37m 1s", "remaining_time": "5h 36m 28s"} +{"eps": 0, "objective/kl": 35.46149445, "objective/entropy": -119.84375763, "objective/non_score_reward": -1.77307475, "objective/rlhf_reward": 13.00036335, "objective/scores": 14.75, "policy/approxkl_avg": 0.00084628, "policy/clipfrac_avg": 0.00630799, "loss/policy_avg": -0.02830285, "loss/value_avg": 0.47069874, "val/clipfrac_avg": 0.09303553, "policy/entropy_avg": 0.1598587, "val/ratio": 0.98867607, "val/ratio_var": 8.341e-05, "val/num_eos_tokens": 242, "lr": 3.27e-06, "episode": 12528, "epoch": 0.63272727, "global_step/max_steps": "783/1238", "percentage": "63.25%", "elapsed_time": "9h 37m 39s", "remaining_time": "5h 35m 40s"} +{"eps": 0, "objective/kl": 35.0530014, "objective/entropy": -32.83402634, "objective/non_score_reward": -1.75265002, "objective/rlhf_reward": 13.70828724, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00105239, "policy/clipfrac_avg": 0.00591615, "loss/policy_avg": -0.03483297, "loss/value_avg": 0.22785367, "val/clipfrac_avg": 0.00789338, "policy/entropy_avg": 0.21710396, "val/ratio": 0.98596728, "val/ratio_var": 5.139e-05, "val/num_eos_tokens": 463, "lr": 3.26e-06, "episode": 12544, "epoch": 0.63353535, "global_step/max_steps": "784/1238", "percentage": "63.33%", "elapsed_time": "9h 38m 25s", "remaining_time": "5h 34m 57s"} +{"eps": 0, "objective/kl": 41.38114929, "objective/entropy": -109.50154114, "objective/non_score_reward": -2.06905746, "objective/rlhf_reward": 13.82156754, "objective/scores": 15.875, "policy/approxkl_avg": 0.00093203, "policy/clipfrac_avg": 0.00533785, "loss/policy_avg": 0.17171788, "loss/value_avg": 0.59372562, "val/clipfrac_avg": 0.04342216, "policy/entropy_avg": 0.18284678, "val/ratio": 0.98809719, "val/ratio_var": 7.803e-05, "val/num_eos_tokens": 252, "lr": 3.25e-06, "episode": 12560, "epoch": 0.63434343, "global_step/max_steps": "785/1238", "percentage": "63.41%", "elapsed_time": "9h 39m 3s", "remaining_time": "5h 34m 9s"} +{"eps": 0, "objective/kl": 39.33965302, "objective/entropy": -80.62982941, "objective/non_score_reward": -1.96698284, "objective/rlhf_reward": 14.37676716, "objective/scores": 16.375, "policy/approxkl_avg": 0.00097266, "policy/clipfrac_avg": 0.00586183, "loss/policy_avg": -0.12326153, "loss/value_avg": 0.22453082, "val/clipfrac_avg": 0.04197417, "policy/entropy_avg": 0.1867485, "val/ratio": 0.98672694, "val/ratio_var": 3.261e-05, "val/num_eos_tokens": 55, "lr": 3.24e-06, "episode": 12576, "epoch": 0.63515152, "global_step/max_steps": "786/1238", "percentage": "63.49%", "elapsed_time": "9h 40m 2s", "remaining_time": "5h 33m 33s"} +{"eps": 0, "objective/kl": 46.58558655, "objective/entropy": -64.31414032, "objective/non_score_reward": -2.32927942, "objective/rlhf_reward": 14.5769701, "objective/scores": 16.875, "policy/approxkl_avg": 0.00095467, "policy/clipfrac_avg": 0.00552854, "loss/policy_avg": 0.02891068, "loss/value_avg": 0.2414318, "val/clipfrac_avg": 0.00137579, "policy/entropy_avg": 0.18930054, "val/ratio": 0.98720598, "val/ratio_var": 3.51e-05, "val/num_eos_tokens": 363, "lr": 3.22e-06, "episode": 12592, "epoch": 0.6359596, "global_step/max_steps": "787/1238", "percentage": "63.57%", "elapsed_time": "9h 40m 55s", "remaining_time": "5h 32m 54s"} +{"eps": 0, "objective/kl": 42.85956573, "objective/entropy": -57.81846237, "objective/non_score_reward": -2.14297843, "objective/rlhf_reward": 12.14999008, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00111358, "policy/clipfrac_avg": 0.00937896, "loss/policy_avg": -0.19538097, "loss/value_avg": 0.71605957, "val/clipfrac_avg": 0.16906279, "policy/entropy_avg": 0.20344543, "val/ratio": 0.98616219, "val/ratio_var": 9.242e-05, "val/num_eos_tokens": 481, "lr": 3.21e-06, "episode": 12608, "epoch": 0.63676768, "global_step/max_steps": "788/1238", "percentage": "63.65%", "elapsed_time": "9h 41m 29s", "remaining_time": "5h 32m 4s"} +{"eps": 0, "objective/kl": 36.84936905, "objective/entropy": -76.67124939, "objective/non_score_reward": -1.84246862, "objective/rlhf_reward": 12.25128174, "objective/scores": 14.125, "policy/approxkl_avg": 0.00099526, "policy/clipfrac_avg": 0.00415037, "loss/policy_avg": 0.04801831, "loss/value_avg": 0.43121871, "val/clipfrac_avg": 0.08804324, "policy/entropy_avg": 0.18071389, "val/ratio": 0.98674035, "val/ratio_var": 0.00013289, "val/num_eos_tokens": 357, "lr": 3.2e-06, "episode": 12624, "epoch": 0.63757576, "global_step/max_steps": "789/1238", "percentage": "63.73%", "elapsed_time": "9h 41m 59s", "remaining_time": "5h 31m 11s"} +{"eps": 0, "objective/kl": 33.96638489, "objective/entropy": -126.28926086, "objective/non_score_reward": -1.69831896, "objective/rlhf_reward": 12.97355652, "objective/scores": 14.6875, "policy/approxkl_avg": 0.00073027, "policy/clipfrac_avg": 0.00839798, "loss/policy_avg": 0.18971527, "loss/value_avg": 0.66260362, "val/clipfrac_avg": 0.06916167, "policy/entropy_avg": 0.14786386, "val/ratio": 0.99105078, "val/ratio_var": 5.284e-05, "val/num_eos_tokens": 600, "lr": 3.19e-06, "episode": 12640, "epoch": 0.63838384, "global_step/max_steps": "790/1238", "percentage": "63.81%", "elapsed_time": "9h 42m 34s", "remaining_time": "5h 30m 22s"} +{"eps": 0, "objective/kl": 40.16947556, "objective/entropy": -75.64164734, "objective/non_score_reward": -2.00847387, "objective/rlhf_reward": 12.8665266, "objective/scores": 14.875, "policy/approxkl_avg": 0.00088978, "policy/clipfrac_avg": 0.00454002, "loss/policy_avg": -0.42793322, "loss/value_avg": 0.51179576, "val/clipfrac_avg": 0.03385417, "policy/entropy_avg": 0.18616325, "val/ratio": 0.98735023, "val/ratio_var": 6.439e-05, "val/num_eos_tokens": 224, "lr": 3.17e-06, "episode": 12656, "epoch": 0.63919192, "global_step/max_steps": "791/1238", "percentage": "63.89%", "elapsed_time": "9h 43m 14s", "remaining_time": "5h 29m 35s"} +{"eps": 0, "objective/kl": 35.08636475, "objective/entropy": -36.10865402, "objective/non_score_reward": -1.75431824, "objective/rlhf_reward": 14.99568176, "objective/scores": 16.75, "policy/approxkl_avg": 0.00101644, "policy/clipfrac_avg": 0.00334266, "loss/policy_avg": -0.30223173, "loss/value_avg": 0.74147612, "val/clipfrac_avg": 0.08375495, "policy/entropy_avg": 0.20384598, "val/ratio": 0.98627621, "val/ratio_var": 8.216e-05, "val/num_eos_tokens": 121, "lr": 3.16e-06, "episode": 12672, "epoch": 0.64, "global_step/max_steps": "792/1238", "percentage": "63.97%", "elapsed_time": "9h 43m 51s", "remaining_time": "5h 28m 47s"} +{"eps": 0, "objective/kl": 33.01978302, "objective/entropy": -52.89218521, "objective/non_score_reward": -1.65098929, "objective/rlhf_reward": 12.27088547, "objective/scores": 13.9375, "policy/approxkl_avg": 0.00096372, "policy/clipfrac_avg": 0.00659569, "loss/policy_avg": 0.1155612, "loss/value_avg": 0.44696569, "val/clipfrac_avg": 0.03139581, "policy/entropy_avg": 0.1855135, "val/ratio": 0.98755574, "val/ratio_var": 4.978e-05, "val/num_eos_tokens": 483, "lr": 3.15e-06, "episode": 12688, "epoch": 0.64080808, "global_step/max_steps": "793/1238", "percentage": "64.05%", "elapsed_time": "9h 44m 20s", "remaining_time": "5h 27m 54s"} +{"eps": 0, "objective/kl": 38.74113846, "objective/entropy": -44.28551102, "objective/non_score_reward": -1.93705702, "objective/rlhf_reward": 13.5316925, "objective/scores": 15.5, "policy/approxkl_avg": 0.00098926, "policy/clipfrac_avg": 0.00600271, "loss/policy_avg": -0.11985204, "loss/value_avg": 0.55953181, "val/clipfrac_avg": 0.05136706, "policy/entropy_avg": 0.20067406, "val/ratio": 0.98707116, "val/ratio_var": 6.21e-05, "val/num_eos_tokens": 146, "lr": 3.14e-06, "episode": 12704, "epoch": 0.64161616, "global_step/max_steps": "794/1238", "percentage": "64.14%", "elapsed_time": "9h 44m 57s", "remaining_time": "5h 27m 6s"} +{"eps": 0, "objective/kl": 36.60810852, "objective/entropy": -47.06381226, "objective/non_score_reward": -1.83040547, "objective/rlhf_reward": 13.66178226, "objective/scores": 15.5, "policy/approxkl_avg": 0.00099834, "policy/clipfrac_avg": 0.00418364, "loss/policy_avg": 0.06161116, "loss/value_avg": 0.20387179, "val/clipfrac_avg": 0.08834638, "policy/entropy_avg": 0.19140625, "val/ratio": 0.98746288, "val/ratio_var": 3.379e-05, "val/num_eos_tokens": 342, "lr": 3.12e-06, "episode": 12720, "epoch": 0.64242424, "global_step/max_steps": "795/1238", "percentage": "64.22%", "elapsed_time": "9h 45m 31s", "remaining_time": "5h 26m 16s"} +{"eps": 0, "objective/kl": 35.12114716, "objective/entropy": -104.79203796, "objective/non_score_reward": -1.7560575, "objective/rlhf_reward": 13.48612976, "objective/scores": 15.25, "policy/approxkl_avg": 0.00081466, "policy/clipfrac_avg": 0.00569065, "loss/policy_avg": 0.02652082, "loss/value_avg": 0.44328654, "val/clipfrac_avg": 0.06147324, "policy/entropy_avg": 0.15022373, "val/ratio": 0.98989463, "val/ratio_var": 4.619e-05, "val/num_eos_tokens": 748, "lr": 3.11e-06, "episode": 12736, "epoch": 0.64323232, "global_step/max_steps": "796/1238", "percentage": "64.30%", "elapsed_time": "9h 46m 16s", "remaining_time": "5h 25m 32s"} +{"eps": 0, "objective/kl": 38.22411728, "objective/entropy": -88.317276, "objective/non_score_reward": -1.91120601, "objective/rlhf_reward": 14.69816971, "objective/scores": 16.625, "policy/approxkl_avg": 0.00082006, "policy/clipfrac_avg": 0.00769325, "loss/policy_avg": 0.06834742, "loss/value_avg": 0.57316518, "val/clipfrac_avg": 0.11237858, "policy/entropy_avg": 0.16072083, "val/ratio": 0.99055046, "val/ratio_var": 4.707e-05, "val/num_eos_tokens": 240, "lr": 3.1e-06, "episode": 12752, "epoch": 0.6440404, "global_step/max_steps": "797/1238", "percentage": "64.38%", "elapsed_time": "9h 46m 50s", "remaining_time": "5h 24m 42s"} +{"eps": 0, "objective/kl": 29.70239258, "objective/entropy": -124.79135895, "objective/non_score_reward": -1.48511958, "objective/rlhf_reward": 13.68675423, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00087824, "policy/clipfrac_avg": 0.00598435, "loss/policy_avg": -0.09761235, "loss/value_avg": 0.70889091, "val/clipfrac_avg": 0.03064167, "policy/entropy_avg": 0.16924286, "val/ratio": 0.98826355, "val/ratio_var": 7.516e-05, "val/num_eos_tokens": 454, "lr": 3.09e-06, "episode": 12768, "epoch": 0.64484848, "global_step/max_steps": "798/1238", "percentage": "64.46%", "elapsed_time": "9h 47m 31s", "remaining_time": "5h 23m 56s"} +{"eps": 0, "objective/kl": 25.74894142, "objective/entropy": -101.50771332, "objective/non_score_reward": -1.28744709, "objective/rlhf_reward": 13.73599052, "objective/scores": 15.0, "policy/approxkl_avg": 0.00075122, "policy/clipfrac_avg": 0.01198269, "loss/policy_avg": 0.14146611, "loss/value_avg": 0.73210126, "val/clipfrac_avg": 0.01267799, "policy/entropy_avg": 0.15892601, "val/ratio": 0.99101341, "val/ratio_var": 6.09e-05, "val/num_eos_tokens": 12, "lr": 3.07e-06, "episode": 12784, "epoch": 0.64565657, "global_step/max_steps": "799/1238", "percentage": "64.54%", "elapsed_time": "9h 48m 8s", "remaining_time": "5h 23m 8s"} +{"eps": 0, "objective/kl": 34.02339935, "objective/entropy": -68.88214874, "objective/non_score_reward": -1.70116997, "objective/rlhf_reward": 12.49414253, "objective/scores": 14.1875, "policy/approxkl_avg": 0.00102393, "policy/clipfrac_avg": 0.01089456, "loss/policy_avg": -0.04285183, "loss/value_avg": 0.34527594, "val/clipfrac_avg": 0.02672293, "policy/entropy_avg": 0.17738628, "val/ratio": 0.98885953, "val/ratio_var": 6.889e-05, "val/num_eos_tokens": 67, "lr": 3.06e-06, "episode": 12800, "epoch": 0.64646465, "global_step/max_steps": "800/1238", "percentage": "64.62%", "elapsed_time": "9h 48m 37s", "remaining_time": "5h 22m 16s"} +{"eps": 0, "objective/kl": 32.48891068, "objective/entropy": -89.62500763, "objective/non_score_reward": -1.62444556, "objective/rlhf_reward": 14.22711658, "objective/scores": 15.875, "policy/approxkl_avg": 0.00093855, "policy/clipfrac_avg": 0.00492091, "loss/policy_avg": -0.05754969, "loss/value_avg": 0.40233207, "val/clipfrac_avg": 0.03176931, "policy/entropy_avg": 0.1744045, "val/ratio": 0.98802233, "val/ratio_var": 5.388e-05, "val/num_eos_tokens": 9, "lr": 3.05e-06, "episode": 12816, "epoch": 0.64727273, "global_step/max_steps": "801/1238", "percentage": "64.70%", "elapsed_time": "9h 49m 37s", "remaining_time": "5h 21m 40s"} +{"eps": 0, "objective/kl": 40.6991272, "objective/entropy": -89.31193542, "objective/non_score_reward": -2.03495646, "objective/rlhf_reward": 12.16035557, "objective/scores": 14.1875, "policy/approxkl_avg": 0.00086792, "policy/clipfrac_avg": 0.00482267, "loss/policy_avg": 0.24831016, "loss/value_avg": 0.23584425, "val/clipfrac_avg": 0.05813269, "policy/entropy_avg": 0.16604805, "val/ratio": 0.98901653, "val/ratio_var": 5.875e-05, "val/num_eos_tokens": 149, "lr": 3.04e-06, "episode": 12832, "epoch": 0.64808081, "global_step/max_steps": "802/1238", "percentage": "64.78%", "elapsed_time": "9h 50m 17s", "remaining_time": "5h 20m 54s"} +{"eps": 0, "objective/kl": 38.51166916, "objective/entropy": -148.43241882, "objective/non_score_reward": -1.9255836, "objective/rlhf_reward": 11.91816711, "objective/scores": 13.875, "policy/approxkl_avg": 0.00079626, "policy/clipfrac_avg": 0.0040262, "loss/policy_avg": -0.10593897, "loss/value_avg": 0.54564345, "val/clipfrac_avg": 0.04293014, "policy/entropy_avg": 0.16094494, "val/ratio": 0.98935604, "val/ratio_var": 6.858e-05, "val/num_eos_tokens": 331, "lr": 3.03e-06, "episode": 12848, "epoch": 0.64888889, "global_step/max_steps": "803/1238", "percentage": "64.86%", "elapsed_time": "9h 51m 16s", "remaining_time": "5h 20m 18s"} +{"eps": 0, "objective/kl": 40.92150497, "objective/entropy": -115.89801788, "objective/non_score_reward": -2.04607511, "objective/rlhf_reward": 12.86798763, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00086648, "policy/clipfrac_avg": 0.00966357, "loss/policy_avg": 0.06602153, "loss/value_avg": 0.28230047, "val/clipfrac_avg": 0.03888743, "policy/entropy_avg": 0.1512146, "val/ratio": 0.99006015, "val/ratio_var": 5.009e-05, "val/num_eos_tokens": 224, "lr": 3.01e-06, "episode": 12864, "epoch": 0.64969697, "global_step/max_steps": "804/1238", "percentage": "64.94%", "elapsed_time": "9h 52m 21s", "remaining_time": "5h 19m 45s"} +{"eps": 0, "objective/kl": 35.05690765, "objective/entropy": -77.7519989, "objective/non_score_reward": -1.75284541, "objective/rlhf_reward": 14.00496769, "objective/scores": 15.75, "policy/approxkl_avg": 0.00102396, "policy/clipfrac_avg": 0.00462991, "loss/policy_avg": -0.39892638, "loss/value_avg": 0.42402726, "val/clipfrac_avg": 0.07910473, "policy/entropy_avg": 0.20476627, "val/ratio": 0.98612338, "val/ratio_var": 8.373e-05, "val/num_eos_tokens": 219, "lr": 3e-06, "episode": 12880, "epoch": 0.65050505, "global_step/max_steps": "805/1238", "percentage": "65.02%", "elapsed_time": "9h 53m 0s", "remaining_time": "5h 18m 58s"} +{"eps": 0, "objective/kl": 40.88092804, "objective/entropy": -85.76292419, "objective/non_score_reward": -2.0440464, "objective/rlhf_reward": 14.1747036, "objective/scores": 16.25, "policy/approxkl_avg": 0.00100818, "policy/clipfrac_avg": 0.00412846, "loss/policy_avg": -0.23323669, "loss/value_avg": 0.46074694, "val/clipfrac_avg": 0.16490784, "policy/entropy_avg": 0.19212532, "val/ratio": 0.98758388, "val/ratio_var": 6.451e-05, "val/num_eos_tokens": 249, "lr": 2.99e-06, "episode": 12896, "epoch": 0.65131313, "global_step/max_steps": "806/1238", "percentage": "65.11%", "elapsed_time": "9h 53m 35s", "remaining_time": "5h 18m 9s"} +{"eps": 0, "objective/kl": 33.47719955, "objective/entropy": -62.67930222, "objective/non_score_reward": -1.67386007, "objective/rlhf_reward": 12.77145195, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00110917, "policy/clipfrac_avg": 0.00747725, "loss/policy_avg": -0.11845396, "loss/value_avg": 0.45925352, "val/clipfrac_avg": 0.03311447, "policy/entropy_avg": 0.19838142, "val/ratio": 0.98777109, "val/ratio_var": 6.532e-05, "val/num_eos_tokens": 17, "lr": 2.98e-06, "episode": 12912, "epoch": 0.65212121, "global_step/max_steps": "807/1238", "percentage": "65.19%", "elapsed_time": "9h 54m 3s", "remaining_time": "5h 17m 16s"} +{"eps": 0, "objective/kl": 34.25027847, "objective/entropy": -88.94641113, "objective/non_score_reward": -1.71251392, "objective/rlhf_reward": 12.56873608, "objective/scores": 14.25, "policy/approxkl_avg": 0.00094483, "policy/clipfrac_avg": 0.00638776, "loss/policy_avg": 0.04699004, "loss/value_avg": 0.26954371, "val/clipfrac_avg": 0.01817773, "policy/entropy_avg": 0.18566322, "val/ratio": 0.98820633, "val/ratio_var": 5.754e-05, "val/num_eos_tokens": 761, "lr": 2.96e-06, "episode": 12928, "epoch": 0.65292929, "global_step/max_steps": "808/1238", "percentage": "65.27%", "elapsed_time": "9h 55m 6s", "remaining_time": "5h 16m 42s"} +{"eps": 0, "objective/kl": 40.86418152, "objective/entropy": -93.69944763, "objective/non_score_reward": -2.04320908, "objective/rlhf_reward": 13.54272842, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00079904, "policy/clipfrac_avg": 0.00640861, "loss/policy_avg": 0.00823296, "loss/value_avg": 0.44794196, "val/clipfrac_avg": 0.08976449, "policy/entropy_avg": 0.14969254, "val/ratio": 0.98978007, "val/ratio_var": 6.073e-05, "val/num_eos_tokens": 215, "lr": 2.95e-06, "episode": 12944, "epoch": 0.65373737, "global_step/max_steps": "809/1238", "percentage": "65.35%", "elapsed_time": "9h 55m 38s", "remaining_time": "5h 15m 51s"} +{"eps": 0, "objective/kl": 35.73942184, "objective/entropy": -41.68758774, "objective/non_score_reward": -1.78697133, "objective/rlhf_reward": 15.31459141, "objective/scores": 17.125, "policy/approxkl_avg": 0.0008846, "policy/clipfrac_avg": 0.00692939, "loss/policy_avg": 0.02401058, "loss/value_avg": 0.33788526, "val/clipfrac_avg": 0.00328684, "policy/entropy_avg": 0.17317677, "val/ratio": 0.98928654, "val/ratio_var": 4.36e-05, "val/num_eos_tokens": 327, "lr": 2.94e-06, "episode": 12960, "epoch": 0.65454545, "global_step/max_steps": "810/1238", "percentage": "65.43%", "elapsed_time": "9h 56m 19s", "remaining_time": "5h 15m 5s"} +{"eps": 0, "objective/kl": 36.01569366, "objective/entropy": -45.97090149, "objective/non_score_reward": -1.80078447, "objective/rlhf_reward": 14.23046589, "objective/scores": 16.0, "policy/approxkl_avg": 0.00097621, "policy/clipfrac_avg": 0.00480386, "loss/policy_avg": 0.14561704, "loss/value_avg": 0.51157081, "val/clipfrac_avg": 0.07713419, "policy/entropy_avg": 0.18830299, "val/ratio": 0.98788989, "val/ratio_var": 4.796e-05, "val/num_eos_tokens": 267, "lr": 2.93e-06, "episode": 12976, "epoch": 0.65535354, "global_step/max_steps": "811/1238", "percentage": "65.51%", "elapsed_time": "9h 56m 44s", "remaining_time": "5h 14m 11s"} +{"eps": 0, "objective/kl": 36.48134613, "objective/entropy": -66.14154053, "objective/non_score_reward": -1.82406735, "objective/rlhf_reward": 14.17593288, "objective/scores": 16.0, "policy/approxkl_avg": 0.00098237, "policy/clipfrac_avg": 0.00344618, "loss/policy_avg": 0.10021464, "loss/value_avg": 0.55802929, "val/clipfrac_avg": 0.10748951, "policy/entropy_avg": 0.19554329, "val/ratio": 0.98605931, "val/ratio_var": 5.329e-05, "val/num_eos_tokens": 292, "lr": 2.92e-06, "episode": 12992, "epoch": 0.65616162, "global_step/max_steps": "812/1238", "percentage": "65.59%", "elapsed_time": "9h 57m 27s", "remaining_time": "5h 13m 26s"} +{"eps": 0, "objective/kl": 36.02103424, "objective/entropy": -110.84692383, "objective/non_score_reward": -1.80105162, "objective/rlhf_reward": 16.33176041, "objective/scores": 18.125, "policy/approxkl_avg": 0.00080352, "policy/clipfrac_avg": 0.00594027, "loss/policy_avg": -0.1050981, "loss/value_avg": 0.46006063, "val/clipfrac_avg": 0.07604548, "policy/entropy_avg": 0.15110207, "val/ratio": 0.98981333, "val/ratio_var": 6.09e-05, "val/num_eos_tokens": 523, "lr": 2.9e-06, "episode": 13008, "epoch": 0.6569697, "global_step/max_steps": "813/1238", "percentage": "65.67%", "elapsed_time": "9h 58m 12s", "remaining_time": "5h 12m 43s"} +{"eps": 0, "objective/kl": 34.03726578, "objective/entropy": -40.75424576, "objective/non_score_reward": -1.70186341, "objective/rlhf_reward": 13.09110451, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00098038, "policy/clipfrac_avg": 0.00842619, "loss/policy_avg": -0.11711955, "loss/value_avg": 1.21425283, "val/clipfrac_avg": 0.0297619, "policy/entropy_avg": 0.20235157, "val/ratio": 0.98713148, "val/ratio_var": 6.276e-05, "val/num_eos_tokens": 113, "lr": 2.89e-06, "episode": 13024, "epoch": 0.65777778, "global_step/max_steps": "814/1238", "percentage": "65.75%", "elapsed_time": "9h 58m 45s", "remaining_time": "5h 11m 52s"} +{"eps": 0, "objective/kl": 42.53788757, "objective/entropy": -141.41366577, "objective/non_score_reward": -2.12689447, "objective/rlhf_reward": 13.5371685, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00087684, "policy/clipfrac_avg": 0.00673226, "loss/policy_avg": -0.1814366, "loss/value_avg": 0.57157671, "val/clipfrac_avg": 0.03276198, "policy/entropy_avg": 0.17724037, "val/ratio": 0.98854804, "val/ratio_var": 8.229e-05, "val/num_eos_tokens": 221, "lr": 2.88e-06, "episode": 13040, "epoch": 0.65858586, "global_step/max_steps": "815/1238", "percentage": "65.83%", "elapsed_time": "9h 59m 45s", "remaining_time": "5h 11m 17s"} +{"eps": 0, "objective/kl": 35.61436844, "objective/entropy": -61.10264587, "objective/non_score_reward": -1.78071833, "objective/rlhf_reward": 14.39115715, "objective/scores": 16.125, "policy/approxkl_avg": 0.00091949, "policy/clipfrac_avg": 0.01862441, "loss/policy_avg": 0.30176339, "loss/value_avg": 0.32543865, "val/clipfrac_avg": 0.00731286, "policy/entropy_avg": 0.14903784, "val/ratio": 0.98968929, "val/ratio_var": 5.834e-05, "val/num_eos_tokens": 112, "lr": 2.87e-06, "episode": 13056, "epoch": 0.65939394, "global_step/max_steps": "816/1238", "percentage": "65.91%", "elapsed_time": "10h 0m 19s", "remaining_time": "5h 10m 27s"} +{"eps": 0, "objective/kl": 34.15174103, "objective/entropy": -117.86632538, "objective/non_score_reward": -1.707587, "objective/rlhf_reward": 14.51116276, "objective/scores": 16.25, "policy/approxkl_avg": 0.00095056, "policy/clipfrac_avg": 0.00845101, "loss/policy_avg": 0.06899265, "loss/value_avg": 0.69894552, "val/clipfrac_avg": 0.00058962, "policy/entropy_avg": 0.16999722, "val/ratio": 0.98763144, "val/ratio_var": 9.733e-05, "val/num_eos_tokens": 435, "lr": 2.85e-06, "episode": 13072, "epoch": 0.66020202, "global_step/max_steps": "817/1238", "percentage": "65.99%", "elapsed_time": "10h 1m 4s", "remaining_time": "5h 9m 44s"} +{"eps": 0, "objective/kl": 40.51255035, "objective/entropy": -52.28473663, "objective/non_score_reward": -2.02562761, "objective/rlhf_reward": 14.45874786, "objective/scores": 16.5, "policy/approxkl_avg": 0.00088451, "policy/clipfrac_avg": 0.00509411, "loss/policy_avg": -0.05428571, "loss/value_avg": 0.29283434, "val/clipfrac_avg": 0.09271319, "policy/entropy_avg": 0.16685867, "val/ratio": 0.9885065, "val/ratio_var": 4.854e-05, "val/num_eos_tokens": 32, "lr": 2.84e-06, "episode": 13088, "epoch": 0.6610101, "global_step/max_steps": "818/1238", "percentage": "66.07%", "elapsed_time": "10h 1m 42s", "remaining_time": "5h 8m 56s"} +{"eps": 0, "objective/kl": 42.01267242, "objective/entropy": -196.32389832, "objective/non_score_reward": -2.10063362, "objective/rlhf_reward": 12.51655388, "objective/scores": 14.625, "policy/approxkl_avg": 0.00079045, "policy/clipfrac_avg": 0.00641987, "loss/policy_avg": 0.12085427, "loss/value_avg": 0.56701958, "val/clipfrac_avg": 0.05388574, "policy/entropy_avg": 0.1437664, "val/ratio": 0.98991936, "val/ratio_var": 4.293e-05, "val/num_eos_tokens": 496, "lr": 2.83e-06, "episode": 13104, "epoch": 0.66181818, "global_step/max_steps": "819/1238", "percentage": "66.16%", "elapsed_time": "10h 2m 49s", "remaining_time": "5h 8m 24s"} +{"eps": 0, "objective/kl": 38.737854, "objective/entropy": -58.51355362, "objective/non_score_reward": -1.93689275, "objective/rlhf_reward": 12.88341904, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00102885, "policy/clipfrac_avg": 0.02149814, "loss/policy_avg": 0.60588443, "loss/value_avg": 0.72947872, "val/clipfrac_avg": 0.0723545, "policy/entropy_avg": 0.20896149, "val/ratio": 0.98879695, "val/ratio_var": 4.814e-05, "val/num_eos_tokens": 134, "lr": 2.82e-06, "episode": 13120, "epoch": 0.66262626, "global_step/max_steps": "820/1238", "percentage": "66.24%", "elapsed_time": "10h 3m 25s", "remaining_time": "5h 7m 36s"} +{"eps": 0, "objective/kl": 33.36050034, "objective/entropy": -38.43750763, "objective/non_score_reward": -1.66802514, "objective/rlhf_reward": 13.54291248, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00106981, "policy/clipfrac_avg": 0.00493103, "loss/policy_avg": -0.2193266, "loss/value_avg": 0.45023149, "val/clipfrac_avg": 0.0085249, "policy/entropy_avg": 0.19058466, "val/ratio": 0.98641014, "val/ratio_var": 5.521e-05, "val/num_eos_tokens": 190, "lr": 2.81e-06, "episode": 13136, "epoch": 0.66343434, "global_step/max_steps": "821/1238", "percentage": "66.32%", "elapsed_time": "10h 3m 55s", "remaining_time": "5h 6m 44s"} +{"eps": 0, "objective/kl": 44.5791626, "objective/entropy": 13.10763931, "objective/non_score_reward": -2.22895813, "objective/rlhf_reward": 13.36479282, "objective/scores": 15.625, "policy/approxkl_avg": 0.00140382, "policy/clipfrac_avg": 0.00650985, "loss/policy_avg": -0.01703065, "loss/value_avg": 0.29020083, "val/clipfrac_avg": 0.03128276, "policy/entropy_avg": 0.26366425, "val/ratio": 0.98244882, "val/ratio_var": 4.771e-05, "val/num_eos_tokens": 35, "lr": 2.79e-06, "episode": 13152, "epoch": 0.66424242, "global_step/max_steps": "822/1238", "percentage": "66.40%", "elapsed_time": "10h 4m 27s", "remaining_time": "5h 5m 54s"} +{"eps": 0, "objective/kl": 42.75294495, "objective/entropy": -94.84840393, "objective/non_score_reward": -2.13764763, "objective/rlhf_reward": 12.26860237, "objective/scores": 14.375, "policy/approxkl_avg": 0.00104913, "policy/clipfrac_avg": 0.00498528, "loss/policy_avg": -0.02819274, "loss/value_avg": 0.28618735, "val/clipfrac_avg": 0.00754885, "policy/entropy_avg": 0.2000885, "val/ratio": 0.98667419, "val/ratio_var": 4.918e-05, "val/num_eos_tokens": 289, "lr": 2.78e-06, "episode": 13168, "epoch": 0.66505051, "global_step/max_steps": "823/1238", "percentage": "66.48%", "elapsed_time": "10h 5m 33s", "remaining_time": "5h 5m 21s"} +{"eps": 0, "objective/kl": 45.08838654, "objective/entropy": -104.75354004, "objective/non_score_reward": -2.25441933, "objective/rlhf_reward": 12.21433067, "objective/scores": 14.5, "policy/approxkl_avg": 0.00091553, "policy/clipfrac_avg": 0.00883184, "loss/policy_avg": -0.02481849, "loss/value_avg": 0.36116296, "val/clipfrac_avg": 0.00242584, "policy/entropy_avg": 0.16748524, "val/ratio": 0.98903751, "val/ratio_var": 5.131e-05, "val/num_eos_tokens": 163, "lr": 2.77e-06, "episode": 13184, "epoch": 0.66585859, "global_step/max_steps": "824/1238", "percentage": "66.56%", "elapsed_time": "10h 6m 17s", "remaining_time": "5h 4m 36s"} +{"eps": 0, "objective/kl": 30.59856606, "objective/entropy": -74.55836487, "objective/non_score_reward": -1.52992845, "objective/rlhf_reward": 15.32944679, "objective/scores": 16.875, "policy/approxkl_avg": 0.00083303, "policy/clipfrac_avg": 0.01046492, "loss/policy_avg": -0.28224269, "loss/value_avg": 0.87201047, "val/clipfrac_avg": 0.08580771, "policy/entropy_avg": 0.15100551, "val/ratio": 0.98962355, "val/ratio_var": 9.329e-05, "val/num_eos_tokens": 460, "lr": 2.76e-06, "episode": 13200, "epoch": 0.66666667, "global_step/max_steps": "825/1238", "percentage": "66.64%", "elapsed_time": "10h 6m 42s", "remaining_time": "5h 3m 43s"} +{"eps": 0, "objective/kl": 41.76022339, "objective/entropy": -43.89616394, "objective/non_score_reward": -2.08801126, "objective/rlhf_reward": 13.87292671, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00103179, "policy/clipfrac_avg": 0.0054151, "loss/policy_avg": -0.17151289, "loss/value_avg": 0.45403379, "val/clipfrac_avg": 0.03631856, "policy/entropy_avg": 0.19500446, "val/ratio": 0.98746163, "val/ratio_var": 6.215e-05, "val/num_eos_tokens": 334, "lr": 2.75e-06, "episode": 13216, "epoch": 0.66747475, "global_step/max_steps": "826/1238", "percentage": "66.72%", "elapsed_time": "10h 7m 22s", "remaining_time": "5h 2m 57s"} +{"eps": 0, "objective/kl": 34.05264664, "objective/entropy": -13.07168388, "objective/non_score_reward": -1.70263243, "objective/rlhf_reward": 14.6567421, "objective/scores": 16.375, "policy/approxkl_avg": 0.00125336, "policy/clipfrac_avg": 0.0051752, "loss/policy_avg": -0.0777026, "loss/value_avg": 0.65766162, "val/clipfrac_avg": 0.09026824, "policy/entropy_avg": 0.21735764, "val/ratio": 0.98630142, "val/ratio_var": 0.00010489, "val/num_eos_tokens": 468, "lr": 2.73e-06, "episode": 13232, "epoch": 0.66828283, "global_step/max_steps": "827/1238", "percentage": "66.80%", "elapsed_time": "10h 7m 51s", "remaining_time": "5h 2m 5s"} +{"eps": 0, "objective/kl": 37.01185226, "objective/entropy": -132.30560303, "objective/non_score_reward": -1.85059261, "objective/rlhf_reward": 15.28221893, "objective/scores": 17.125, "policy/approxkl_avg": 0.00083366, "policy/clipfrac_avg": 0.00303435, "loss/policy_avg": -0.11604562, "loss/value_avg": 0.18491808, "val/clipfrac_avg": 0.07020965, "policy/entropy_avg": 0.16221046, "val/ratio": 0.98949653, "val/ratio_var": 4.505e-05, "val/num_eos_tokens": 41, "lr": 2.72e-06, "episode": 13248, "epoch": 0.66909091, "global_step/max_steps": "828/1238", "percentage": "66.88%", "elapsed_time": "10h 8m 45s", "remaining_time": "5h 1m 26s"} +{"eps": 0, "objective/kl": 42.80398941, "objective/entropy": -83.5711441, "objective/non_score_reward": -2.14019942, "objective/rlhf_reward": 13.55511284, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00084909, "policy/clipfrac_avg": 0.02421154, "loss/policy_avg": -0.2308235, "loss/value_avg": 0.93389094, "val/clipfrac_avg": 0.01703278, "policy/entropy_avg": 0.16208172, "val/ratio": 0.98897517, "val/ratio_var": 6.55e-05, "val/num_eos_tokens": 407, "lr": 2.71e-06, "episode": 13264, "epoch": 0.66989899, "global_step/max_steps": "829/1238", "percentage": "66.96%", "elapsed_time": "10h 9m 20s", "remaining_time": "5h 0m 37s"} +{"eps": 0, "objective/kl": 30.28129005, "objective/entropy": -91.5272522, "objective/non_score_reward": -1.51406455, "objective/rlhf_reward": 13.71249771, "objective/scores": 15.25, "policy/approxkl_avg": 0.00064659, "policy/clipfrac_avg": 0.00367457, "loss/policy_avg": -0.17908901, "loss/value_avg": 0.94023603, "val/clipfrac_avg": 0.05747377, "policy/entropy_avg": 0.14063799, "val/ratio": 0.99190259, "val/ratio_var": 6.961e-05, "val/num_eos_tokens": 250, "lr": 2.7e-06, "episode": 13280, "epoch": 0.67070707, "global_step/max_steps": "830/1238", "percentage": "67.04%", "elapsed_time": "10h 9m 48s", "remaining_time": "4h 59m 45s"} +{"eps": 0, "objective/kl": 44.12242889, "objective/entropy": -54.39606476, "objective/non_score_reward": -2.20612144, "objective/rlhf_reward": 14.00481606, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009419, "policy/clipfrac_avg": 0.00952383, "loss/policy_avg": 0.34006393, "loss/value_avg": 1.96887624, "val/clipfrac_avg": 0.0453462, "policy/entropy_avg": 0.18965662, "val/ratio": 0.98809969, "val/ratio_var": 6.566e-05, "val/num_eos_tokens": 245, "lr": 2.69e-06, "episode": 13296, "epoch": 0.67151515, "global_step/max_steps": "831/1238", "percentage": "67.12%", "elapsed_time": "10h 10m 26s", "remaining_time": "4h 58m 58s"} +{"eps": 0, "objective/kl": 38.93806839, "objective/entropy": -66.36386108, "objective/non_score_reward": -1.94690347, "objective/rlhf_reward": 14.07653427, "objective/scores": 16.0, "policy/approxkl_avg": 0.00086386, "policy/clipfrac_avg": 0.00532273, "loss/policy_avg": 0.11609109, "loss/value_avg": 0.56351614, "val/clipfrac_avg": 0.0438162, "policy/entropy_avg": 0.1515255, "val/ratio": 0.98893631, "val/ratio_var": 3.003e-05, "val/num_eos_tokens": 20, "lr": 2.68e-06, "episode": 13312, "epoch": 0.67232323, "global_step/max_steps": "832/1238", "percentage": "67.21%", "elapsed_time": "10h 10m 58s", "remaining_time": "4h 58m 8s"} +{"eps": 0, "objective/kl": 47.16798401, "objective/entropy": -111.39958954, "objective/non_score_reward": -2.35839939, "objective/rlhf_reward": 11.71972656, "objective/scores": 14.0625, "policy/approxkl_avg": 0.00094539, "policy/clipfrac_avg": 0.00460306, "loss/policy_avg": -0.03308401, "loss/value_avg": 0.3210277, "val/clipfrac_avg": 0.06254166, "policy/entropy_avg": 0.18264532, "val/ratio": 0.98794019, "val/ratio_var": 6.488e-05, "val/num_eos_tokens": 763, "lr": 2.66e-06, "episode": 13328, "epoch": 0.67313131, "global_step/max_steps": "833/1238", "percentage": "67.29%", "elapsed_time": "10h 11m 45s", "remaining_time": "4h 57m 26s"} +{"eps": 0, "objective/kl": 37.68748474, "objective/entropy": -69.66492462, "objective/non_score_reward": -1.88437414, "objective/rlhf_reward": 12.76406384, "objective/scores": 14.625, "policy/approxkl_avg": 0.00100554, "policy/clipfrac_avg": 0.00434945, "loss/policy_avg": 0.05182506, "loss/value_avg": 0.82461607, "val/clipfrac_avg": 0.06374437, "policy/entropy_avg": 0.18214893, "val/ratio": 0.98671257, "val/ratio_var": 9.083e-05, "val/num_eos_tokens": 38, "lr": 2.65e-06, "episode": 13344, "epoch": 0.67393939, "global_step/max_steps": "834/1238", "percentage": "67.37%", "elapsed_time": "10h 12m 18s", "remaining_time": "4h 56m 36s"} +{"eps": 0, "objective/kl": 39.28848267, "objective/entropy": -108.79554749, "objective/non_score_reward": -1.96442413, "objective/rlhf_reward": 14.50432587, "objective/scores": 16.5, "policy/approxkl_avg": 0.00106592, "policy/clipfrac_avg": 0.00570767, "loss/policy_avg": -0.04355838, "loss/value_avg": 0.60937959, "val/clipfrac_avg": 0.01337134, "policy/entropy_avg": 0.19863987, "val/ratio": 0.9869523, "val/ratio_var": 8.709e-05, "val/num_eos_tokens": 647, "lr": 2.64e-06, "episode": 13360, "epoch": 0.67474747, "global_step/max_steps": "835/1238", "percentage": "67.45%", "elapsed_time": "10h 12m 54s", "remaining_time": "4h 55m 48s"} +{"eps": 0, "objective/kl": 42.06044388, "objective/entropy": -59.19080734, "objective/non_score_reward": -2.1030221, "objective/rlhf_reward": 13.84229088, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00101653, "policy/clipfrac_avg": 0.00735408, "loss/policy_avg": 0.09040576, "loss/value_avg": 0.45177257, "val/clipfrac_avg": 0.03018788, "policy/entropy_avg": 0.20728302, "val/ratio": 0.98650277, "val/ratio_var": 5.405e-05, "val/num_eos_tokens": 477, "lr": 2.63e-06, "episode": 13376, "epoch": 0.67555556, "global_step/max_steps": "836/1238", "percentage": "67.53%", "elapsed_time": "10h 13m 30s", "remaining_time": "4h 55m 0s"} +{"eps": 0, "objective/kl": 43.63647461, "objective/entropy": -99.0164032, "objective/non_score_reward": -2.18182373, "objective/rlhf_reward": 13.08380127, "objective/scores": 15.25, "policy/approxkl_avg": 0.00091765, "policy/clipfrac_avg": 0.00607822, "loss/policy_avg": -0.21371892, "loss/value_avg": 0.26992351, "val/clipfrac_avg": 0.02565178, "policy/entropy_avg": 0.17147636, "val/ratio": 0.9882133, "val/ratio_var": 6.254e-05, "val/num_eos_tokens": 453, "lr": 2.62e-06, "episode": 13392, "epoch": 0.67636364, "global_step/max_steps": "837/1238", "percentage": "67.61%", "elapsed_time": "10h 14m 7s", "remaining_time": "4h 54m 13s"} +{"eps": 0, "objective/kl": 40.30584335, "objective/entropy": -62.21269989, "objective/non_score_reward": -2.01529217, "objective/rlhf_reward": 15.75814438, "objective/scores": 17.75, "policy/approxkl_avg": 0.0009111, "policy/clipfrac_avg": 0.00501776, "loss/policy_avg": 0.08225659, "loss/value_avg": 0.41060317, "val/clipfrac_avg": 0.0028513, "policy/entropy_avg": 0.18220901, "val/ratio": 0.98786199, "val/ratio_var": 3.237e-05, "val/num_eos_tokens": 268, "lr": 2.6e-06, "episode": 13408, "epoch": 0.67717172, "global_step/max_steps": "838/1238", "percentage": "67.69%", "elapsed_time": "10h 14m 51s", "remaining_time": "4h 53m 29s"} +{"eps": 0, "objective/kl": 35.24858093, "objective/entropy": -38.30493164, "objective/non_score_reward": -1.762429, "objective/rlhf_reward": 12.28444576, "objective/scores": 14.0625, "policy/approxkl_avg": 0.00087087, "policy/clipfrac_avg": 0.00564084, "loss/policy_avg": 0.01355072, "loss/value_avg": 0.487683, "val/clipfrac_avg": 0.0298991, "policy/entropy_avg": 0.16802192, "val/ratio": 0.98920101, "val/ratio_var": 5.169e-05, "val/num_eos_tokens": 258, "lr": 2.59e-06, "episode": 13424, "epoch": 0.6779798, "global_step/max_steps": "839/1238", "percentage": "67.77%", "elapsed_time": "10h 15m 16s", "remaining_time": "4h 52m 36s"} +{"eps": 0, "objective/kl": 36.32350922, "objective/entropy": -63.92339706, "objective/non_score_reward": -1.81617522, "objective/rlhf_reward": 12.97288704, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00095891, "policy/clipfrac_avg": 0.00603301, "loss/policy_avg": 0.16233873, "loss/value_avg": 0.35849041, "val/clipfrac_avg": 0.11569126, "policy/entropy_avg": 0.17229271, "val/ratio": 0.98839033, "val/ratio_var": 7.234e-05, "val/num_eos_tokens": 159, "lr": 2.58e-06, "episode": 13440, "epoch": 0.67878788, "global_step/max_steps": "840/1238", "percentage": "67.85%", "elapsed_time": "10h 15m 52s", "remaining_time": "4h 51m 48s"} +{"eps": 0, "objective/kl": 34.90948486, "objective/entropy": -159.03511047, "objective/non_score_reward": -1.74547446, "objective/rlhf_reward": 13.14515018, "objective/scores": 14.875, "policy/approxkl_avg": 0.00070174, "policy/clipfrac_avg": 0.00539606, "loss/policy_avg": -0.29571331, "loss/value_avg": 0.54518294, "val/clipfrac_avg": 0.01905637, "policy/entropy_avg": 0.12675732, "val/ratio": 0.99142152, "val/ratio_var": 8.74e-05, "val/num_eos_tokens": 508, "lr": 2.57e-06, "episode": 13456, "epoch": 0.67959596, "global_step/max_steps": "841/1238", "percentage": "67.93%", "elapsed_time": "10h 16m 51s", "remaining_time": "4h 51m 11s"} +{"eps": 0, "objective/kl": 36.28149033, "objective/entropy": -83.55934906, "objective/non_score_reward": -1.81407452, "objective/rlhf_reward": 14.18592644, "objective/scores": 16.0, "policy/approxkl_avg": 0.00094741, "policy/clipfrac_avg": 0.0048767, "loss/policy_avg": -0.339807, "loss/value_avg": 0.5796597, "val/clipfrac_avg": 0.03151374, "policy/entropy_avg": 0.14034081, "val/ratio": 0.99014449, "val/ratio_var": 6.344e-05, "val/num_eos_tokens": 208, "lr": 2.56e-06, "episode": 13472, "epoch": 0.68040404, "global_step/max_steps": "842/1238", "percentage": "68.01%", "elapsed_time": "10h 17m 29s", "remaining_time": "4h 50m 24s"} +{"eps": 0, "objective/kl": 46.2301178, "objective/entropy": -74.54005432, "objective/non_score_reward": -2.31150579, "objective/rlhf_reward": 12.99318123, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00098849, "policy/clipfrac_avg": 0.00666792, "loss/policy_avg": 0.1184785, "loss/value_avg": 0.30533195, "val/clipfrac_avg": 0.00010081, "policy/entropy_avg": 0.1965847, "val/ratio": 0.98788202, "val/ratio_var": 4.167e-05, "val/num_eos_tokens": 52, "lr": 2.55e-06, "episode": 13488, "epoch": 0.68121212, "global_step/max_steps": "843/1238", "percentage": "68.09%", "elapsed_time": "10h 17m 59s", "remaining_time": "4h 49m 34s"} +{"eps": 0, "objective/kl": 36.79983521, "objective/entropy": -55.13601685, "objective/non_score_reward": -1.83999169, "objective/rlhf_reward": 12.99594593, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00102569, "policy/clipfrac_avg": 0.00953812, "loss/policy_avg": 0.07477365, "loss/value_avg": 0.7172119, "val/clipfrac_avg": 0.07695203, "policy/entropy_avg": 0.1979599, "val/ratio": 0.9861818, "val/ratio_var": 7.032e-05, "val/num_eos_tokens": 573, "lr": 2.53e-06, "episode": 13504, "epoch": 0.6820202, "global_step/max_steps": "844/1238", "percentage": "68.17%", "elapsed_time": "10h 18m 35s", "remaining_time": "4h 48m 46s"} +{"eps": 0, "objective/kl": 41.09418488, "objective/entropy": -164.99487305, "objective/non_score_reward": -2.05470943, "objective/rlhf_reward": 12.36716557, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00068623, "policy/clipfrac_avg": 0.00563342, "loss/policy_avg": -0.01038045, "loss/value_avg": 0.89786208, "val/clipfrac_avg": 0.10761285, "policy/entropy_avg": 0.1361835, "val/ratio": 0.99079967, "val/ratio_var": 4.17e-05, "val/num_eos_tokens": 289, "lr": 2.52e-06, "episode": 13520, "epoch": 0.68282828, "global_step/max_steps": "845/1238", "percentage": "68.26%", "elapsed_time": "10h 20m 17s", "remaining_time": "4h 48m 29s"} +{"eps": 0, "objective/kl": 37.73551941, "objective/entropy": -119.26876068, "objective/non_score_reward": -1.88677597, "objective/rlhf_reward": 13.76947403, "objective/scores": 15.625, "policy/approxkl_avg": 0.00070937, "policy/clipfrac_avg": 0.02827515, "loss/policy_avg": -0.00119421, "loss/value_avg": 0.78056961, "val/clipfrac_avg": 0.0896709, "policy/entropy_avg": 0.14215064, "val/ratio": 0.99119681, "val/ratio_var": 3.499e-05, "val/num_eos_tokens": 760, "lr": 2.51e-06, "episode": 13536, "epoch": 0.68363636, "global_step/max_steps": "846/1238", "percentage": "68.34%", "elapsed_time": "10h 21m 8s", "remaining_time": "4h 47m 48s"} +{"eps": 0, "objective/kl": 34.76280975, "objective/entropy": -89.72827148, "objective/non_score_reward": -1.73814046, "objective/rlhf_reward": 13.92592239, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00090246, "policy/clipfrac_avg": 0.00750113, "loss/policy_avg": -0.00701949, "loss/value_avg": 0.81986272, "val/clipfrac_avg": 0.02837283, "policy/entropy_avg": 0.17042065, "val/ratio": 0.98995125, "val/ratio_var": 7.211e-05, "val/num_eos_tokens": 17, "lr": 2.5e-06, "episode": 13552, "epoch": 0.68444444, "global_step/max_steps": "847/1238", "percentage": "68.42%", "elapsed_time": "10h 21m 42s", "remaining_time": "4h 46m 59s"} +{"eps": 0, "objective/kl": 46.20475769, "objective/entropy": -184.20425415, "objective/non_score_reward": -2.31023836, "objective/rlhf_reward": 11.72882462, "objective/scores": 14.0625, "policy/approxkl_avg": 0.00076387, "policy/clipfrac_avg": 0.00523015, "loss/policy_avg": 0.01510093, "loss/value_avg": 0.25284868, "val/clipfrac_avg": 0.01555096, "policy/entropy_avg": 0.15384865, "val/ratio": 0.98983651, "val/ratio_var": 5.413e-05, "val/num_eos_tokens": 404, "lr": 2.49e-06, "episode": 13568, "epoch": 0.68525253, "global_step/max_steps": "848/1238", "percentage": "68.50%", "elapsed_time": "10h 23m 25s", "remaining_time": "4h 46m 43s"} +{"eps": 0, "objective/kl": 37.31745148, "objective/entropy": -70.96700287, "objective/non_score_reward": -1.86587274, "objective/rlhf_reward": 13.66537762, "objective/scores": 15.5, "policy/approxkl_avg": 0.00088551, "policy/clipfrac_avg": 0.00663753, "loss/policy_avg": -0.13074954, "loss/value_avg": 0.52544826, "val/clipfrac_avg": 0.05051486, "policy/entropy_avg": 0.17204857, "val/ratio": 0.98822123, "val/ratio_var": 5.447e-05, "val/num_eos_tokens": 24, "lr": 2.48e-06, "episode": 13584, "epoch": 0.68606061, "global_step/max_steps": "849/1238", "percentage": "68.58%", "elapsed_time": "10h 23m 58s", "remaining_time": "4h 45m 53s"} +{"eps": 0, "objective/kl": 48.39326477, "objective/entropy": -62.67339325, "objective/non_score_reward": -2.41966319, "objective/rlhf_reward": 13.56471157, "objective/scores": 16.0, "policy/approxkl_avg": 0.00094315, "policy/clipfrac_avg": 0.00486446, "loss/policy_avg": 0.03931019, "loss/value_avg": 0.18585552, "val/clipfrac_avg": 0.01945145, "policy/entropy_avg": 0.1920433, "val/ratio": 0.98726535, "val/ratio_var": 3.195e-05, "val/num_eos_tokens": 36, "lr": 2.47e-06, "episode": 13600, "epoch": 0.68686869, "global_step/max_steps": "850/1238", "percentage": "68.66%", "elapsed_time": "10h 24m 50s", "remaining_time": "4h 45m 13s"} +{"eps": 0, "objective/kl": 35.36459351, "objective/entropy": -76.03438568, "objective/non_score_reward": -1.76822948, "objective/rlhf_reward": 15.07551956, "objective/scores": 16.875, "policy/approxkl_avg": 0.00085448, "policy/clipfrac_avg": 0.00515319, "loss/policy_avg": 0.0797571, "loss/value_avg": 0.47897339, "val/clipfrac_avg": 0.03764946, "policy/entropy_avg": 0.16641998, "val/ratio": 0.98849797, "val/ratio_var": 4.71e-05, "val/num_eos_tokens": 298, "lr": 2.45e-06, "episode": 13616, "epoch": 0.68767677, "global_step/max_steps": "851/1238", "percentage": "68.74%", "elapsed_time": "10h 25m 20s", "remaining_time": "4h 44m 22s"} +{"eps": 0, "objective/kl": 35.15203857, "objective/entropy": -59.74459076, "objective/non_score_reward": -1.7576021, "objective/rlhf_reward": 15.05489731, "objective/scores": 16.75, "policy/approxkl_avg": 0.00097043, "policy/clipfrac_avg": 0.00468879, "loss/policy_avg": -0.10211583, "loss/value_avg": 0.71992385, "val/clipfrac_avg": 0.17230292, "policy/entropy_avg": 0.17905235, "val/ratio": 0.98723745, "val/ratio_var": 4.299e-05, "val/num_eos_tokens": 644, "lr": 2.44e-06, "episode": 13632, "epoch": 0.68848485, "global_step/max_steps": "852/1238", "percentage": "68.82%", "elapsed_time": "10h 26m 1s", "remaining_time": "4h 43m 37s"} +{"eps": 0, "objective/kl": 41.8716011, "objective/entropy": -63.85935211, "objective/non_score_reward": -2.09358001, "objective/rlhf_reward": 16.01579475, "objective/scores": 18.125, "policy/approxkl_avg": 0.0011342, "policy/clipfrac_avg": 0.00555803, "loss/policy_avg": 0.00284434, "loss/value_avg": 0.34213811, "val/clipfrac_avg": 0.0771886, "policy/entropy_avg": 0.21439552, "val/ratio": 0.98588479, "val/ratio_var": 5.448e-05, "val/num_eos_tokens": 400, "lr": 2.43e-06, "episode": 13648, "epoch": 0.68929293, "global_step/max_steps": "853/1238", "percentage": "68.90%", "elapsed_time": "10h 26m 32s", "remaining_time": "4h 42m 47s"} +{"eps": 0, "objective/kl": 39.14732361, "objective/entropy": -138.73922729, "objective/non_score_reward": -1.95736623, "objective/rlhf_reward": 11.54263401, "objective/scores": 13.5, "policy/approxkl_avg": 0.00114136, "policy/clipfrac_avg": 0.00816559, "loss/policy_avg": 0.11498576, "loss/value_avg": 0.9092654, "val/clipfrac_avg": 0.04511755, "policy/entropy_avg": 0.18193865, "val/ratio": 0.98801249, "val/ratio_var": 7.859e-05, "val/num_eos_tokens": 264, "lr": 2.42e-06, "episode": 13664, "epoch": 0.69010101, "global_step/max_steps": "854/1238", "percentage": "68.98%", "elapsed_time": "10h 28m 16s", "remaining_time": "4h 42m 30s"} +{"eps": 0, "objective/kl": 39.87467957, "objective/entropy": -80.86047363, "objective/non_score_reward": -1.993734, "objective/rlhf_reward": 12.85001564, "objective/scores": 14.875, "policy/approxkl_avg": 0.00085447, "policy/clipfrac_avg": 0.00313618, "loss/policy_avg": 0.4261027, "loss/value_avg": 1.03510857, "val/clipfrac_avg": 0.06461639, "policy/entropy_avg": 0.18071175, "val/ratio": 0.98929036, "val/ratio_var": 6.158e-05, "val/num_eos_tokens": 76, "lr": 2.41e-06, "episode": 13680, "epoch": 0.69090909, "global_step/max_steps": "855/1238", "percentage": "69.06%", "elapsed_time": "10h 28m 55s", "remaining_time": "4h 41m 43s"} +{"eps": 0, "objective/kl": 44.5917511, "objective/entropy": -109.00687408, "objective/non_score_reward": -2.22958755, "objective/rlhf_reward": 14.02041245, "objective/scores": 16.25, "policy/approxkl_avg": 0.00083017, "policy/clipfrac_avg": 0.00429514, "loss/policy_avg": 0.35515934, "loss/value_avg": 0.60045308, "val/clipfrac_avg": 0.05787755, "policy/entropy_avg": 0.16374779, "val/ratio": 0.98921096, "val/ratio_var": 3.649e-05, "val/num_eos_tokens": 355, "lr": 2.4e-06, "episode": 13696, "epoch": 0.69171717, "global_step/max_steps": "856/1238", "percentage": "69.14%", "elapsed_time": "10h 29m 44s", "remaining_time": "4h 41m 1s"} +{"eps": 0, "objective/kl": 33.23346329, "objective/entropy": -65.75347137, "objective/non_score_reward": -1.66167319, "objective/rlhf_reward": 12.88520241, "objective/scores": 14.5625, "policy/approxkl_avg": 0.00084454, "policy/clipfrac_avg": 0.00288879, "loss/policy_avg": -0.07683121, "loss/value_avg": 0.75070626, "val/clipfrac_avg": 0.02958414, "policy/entropy_avg": 0.16889977, "val/ratio": 0.98930049, "val/ratio_var": 4.457e-05, "val/num_eos_tokens": 79, "lr": 2.39e-06, "episode": 13712, "epoch": 0.69252525, "global_step/max_steps": "857/1238", "percentage": "69.22%", "elapsed_time": "10h 30m 37s", "remaining_time": "4h 40m 21s"} +{"eps": 0, "objective/kl": 39.99594116, "objective/entropy": -137.21546936, "objective/non_score_reward": -1.99979711, "objective/rlhf_reward": 12.87520218, "objective/scores": 14.875, "policy/approxkl_avg": 0.00093173, "policy/clipfrac_avg": 0.01636841, "loss/policy_avg": -0.04102202, "loss/value_avg": 0.29514134, "val/clipfrac_avg": 0.0260368, "policy/entropy_avg": 0.18145466, "val/ratio": 0.9879334, "val/ratio_var": 7.448e-05, "val/num_eos_tokens": 133, "lr": 2.37e-06, "episode": 13728, "epoch": 0.69333333, "global_step/max_steps": "858/1238", "percentage": "69.31%", "elapsed_time": "10h 32m 20s", "remaining_time": "4h 40m 3s"} +{"eps": 0, "objective/kl": 41.33527756, "objective/entropy": -126.3914032, "objective/non_score_reward": -2.06676388, "objective/rlhf_reward": 15.82386017, "objective/scores": 17.875, "policy/approxkl_avg": 0.0009033, "policy/clipfrac_avg": 0.00826215, "loss/policy_avg": -0.09325989, "loss/value_avg": 0.47780237, "val/clipfrac_avg": 0.00635818, "policy/entropy_avg": 0.18092346, "val/ratio": 0.98817116, "val/ratio_var": 8.43e-05, "val/num_eos_tokens": 198, "lr": 2.36e-06, "episode": 13744, "epoch": 0.69414141, "global_step/max_steps": "859/1238", "percentage": "69.39%", "elapsed_time": "10h 33m 10s", "remaining_time": "4h 39m 21s"} +{"eps": 0, "objective/kl": 36.61549377, "objective/entropy": -82.32437134, "objective/non_score_reward": -1.83077455, "objective/rlhf_reward": 14.49735069, "objective/scores": 16.375, "policy/approxkl_avg": 0.00094057, "policy/clipfrac_avg": 0.00311824, "loss/policy_avg": -0.1992878, "loss/value_avg": 0.44508162, "val/clipfrac_avg": 0.04030742, "policy/entropy_avg": 0.18523979, "val/ratio": 0.98749447, "val/ratio_var": 5.96e-05, "val/num_eos_tokens": 674, "lr": 2.35e-06, "episode": 13760, "epoch": 0.69494949, "global_step/max_steps": "860/1238", "percentage": "69.47%", "elapsed_time": "10h 33m 50s", "remaining_time": "4h 38m 35s"} +{"eps": 0, "objective/kl": 32.36253357, "objective/entropy": -97.76516724, "objective/non_score_reward": -1.61812675, "objective/rlhf_reward": 14.10843658, "objective/scores": 15.75, "policy/approxkl_avg": 0.00087751, "policy/clipfrac_avg": 0.00676325, "loss/policy_avg": -0.00252264, "loss/value_avg": 0.67126536, "val/clipfrac_avg": 0.01572283, "policy/entropy_avg": 0.1599617, "val/ratio": 0.98979354, "val/ratio_var": 4.972e-05, "val/num_eos_tokens": 755, "lr": 2.34e-06, "episode": 13776, "epoch": 0.69575758, "global_step/max_steps": "861/1238", "percentage": "69.55%", "elapsed_time": "10h 34m 32s", "remaining_time": "4h 37m 50s"} +{"eps": 0, "objective/kl": 36.61561584, "objective/entropy": -61.2616806, "objective/non_score_reward": -1.83078086, "objective/rlhf_reward": 12.76296997, "objective/scores": 14.625, "policy/approxkl_avg": 0.00097614, "policy/clipfrac_avg": 0.00268265, "loss/policy_avg": 0.08292508, "loss/value_avg": 0.55372703, "val/clipfrac_avg": 0.04452361, "policy/entropy_avg": 0.22123146, "val/ratio": 0.9871484, "val/ratio_var": 7.335e-05, "val/num_eos_tokens": 56, "lr": 2.33e-06, "episode": 13792, "epoch": 0.69656566, "global_step/max_steps": "862/1238", "percentage": "69.63%", "elapsed_time": "10h 35m 3s", "remaining_time": "4h 37m 0s"} +{"eps": 0, "objective/kl": 36.92033005, "objective/entropy": -76.7585907, "objective/non_score_reward": -1.84601665, "objective/rlhf_reward": 14.87273312, "objective/scores": 16.75, "policy/approxkl_avg": 0.00099353, "policy/clipfrac_avg": 0.00677395, "loss/policy_avg": 0.36910421, "loss/value_avg": 0.9228214, "val/clipfrac_avg": 0.07717608, "policy/entropy_avg": 0.1806345, "val/ratio": 0.98767233, "val/ratio_var": 6.28e-05, "val/num_eos_tokens": 514, "lr": 2.32e-06, "episode": 13808, "epoch": 0.69737374, "global_step/max_steps": "863/1238", "percentage": "69.71%", "elapsed_time": "10h 36m 17s", "remaining_time": "4h 36m 29s"} +{"eps": 0, "objective/kl": 45.52054596, "objective/entropy": -77.51583862, "objective/non_score_reward": -2.2760272, "objective/rlhf_reward": 14.59897232, "objective/scores": 16.875, "policy/approxkl_avg": 0.00084581, "policy/clipfrac_avg": 0.00543943, "loss/policy_avg": -0.21535149, "loss/value_avg": 0.42513978, "val/clipfrac_avg": 0.03808615, "policy/entropy_avg": 0.16998863, "val/ratio": 0.98946911, "val/ratio_var": 6.018e-05, "val/num_eos_tokens": 93, "lr": 2.31e-06, "episode": 13824, "epoch": 0.69818182, "global_step/max_steps": "864/1238", "percentage": "69.79%", "elapsed_time": "10h 36m 44s", "remaining_time": "4h 35m 37s"} +{"eps": 0, "objective/kl": 39.21794891, "objective/entropy": -82.54826355, "objective/non_score_reward": -1.96089733, "objective/rlhf_reward": 14.35941505, "objective/scores": 16.375, "policy/approxkl_avg": 0.00073732, "policy/clipfrac_avg": 0.00397088, "loss/policy_avg": -0.08341408, "loss/value_avg": 0.7828294, "val/clipfrac_avg": 0.1767582, "policy/entropy_avg": 0.14882469, "val/ratio": 0.99052978, "val/ratio_var": 5.841e-05, "val/num_eos_tokens": 134, "lr": 2.29e-06, "episode": 13840, "epoch": 0.6989899, "global_step/max_steps": "865/1238", "percentage": "69.87%", "elapsed_time": "10h 37m 21s", "remaining_time": "4h 34m 50s"} +{"eps": 0, "objective/kl": 34.11535645, "objective/entropy": -121.09165955, "objective/non_score_reward": -1.70576787, "objective/rlhf_reward": 13.33329487, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00092645, "policy/clipfrac_avg": 0.00503988, "loss/policy_avg": 0.02925305, "loss/value_avg": 0.85409981, "val/clipfrac_avg": 0.02329857, "policy/entropy_avg": 0.18099594, "val/ratio": 0.98838562, "val/ratio_var": 8.089e-05, "val/num_eos_tokens": 1353, "lr": 2.28e-06, "episode": 13856, "epoch": 0.69979798, "global_step/max_steps": "866/1238", "percentage": "69.95%", "elapsed_time": "10h 38m 34s", "remaining_time": "4h 34m 18s"} +{"eps": 0, "objective/kl": 36.36531067, "objective/entropy": -49.65830994, "objective/non_score_reward": -1.81826556, "objective/rlhf_reward": 14.49423409, "objective/scores": 16.25, "policy/approxkl_avg": 0.00104298, "policy/clipfrac_avg": 0.0054741, "loss/policy_avg": -0.14982049, "loss/value_avg": 0.4047302, "val/clipfrac_avg": 0.0032526, "policy/entropy_avg": 0.18477345, "val/ratio": 0.98777008, "val/ratio_var": 9.156e-05, "val/num_eos_tokens": 101, "lr": 2.27e-06, "episode": 13872, "epoch": 0.70060606, "global_step/max_steps": "867/1238", "percentage": "70.03%", "elapsed_time": "10h 39m 1s", "remaining_time": "4h 33m 26s"} +{"eps": 0, "objective/kl": 38.64864349, "objective/entropy": -77.34773254, "objective/non_score_reward": -1.93243206, "objective/rlhf_reward": 13.63788033, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00093766, "policy/clipfrac_avg": 0.00623573, "loss/policy_avg": -0.11652799, "loss/value_avg": 0.76698792, "val/clipfrac_avg": 0.05844173, "policy/entropy_avg": 0.19645977, "val/ratio": 0.9879204, "val/ratio_var": 6.414e-05, "val/num_eos_tokens": 310, "lr": 2.26e-06, "episode": 13888, "epoch": 0.70141414, "global_step/max_steps": "868/1238", "percentage": "70.11%", "elapsed_time": "10h 39m 32s", "remaining_time": "4h 32m 37s"} +{"eps": 0, "objective/kl": 48.81157684, "objective/entropy": -163.28765869, "objective/non_score_reward": -2.44057894, "objective/rlhf_reward": 11.70395279, "objective/scores": 14.125, "policy/approxkl_avg": 0.00084498, "policy/clipfrac_avg": 0.00325181, "loss/policy_avg": -0.01632776, "loss/value_avg": 0.43844515, "val/clipfrac_avg": 0.0148018, "policy/entropy_avg": 0.16161728, "val/ratio": 0.98854387, "val/ratio_var": 5.214e-05, "val/num_eos_tokens": 1949, "lr": 2.25e-06, "episode": 13904, "epoch": 0.70222222, "global_step/max_steps": "869/1238", "percentage": "70.19%", "elapsed_time": "10h 41m 18s", "remaining_time": "4h 32m 18s"} +{"eps": 0, "objective/kl": 40.37352371, "objective/entropy": -90.8267746, "objective/non_score_reward": -2.01867604, "objective/rlhf_reward": 15.1531992, "objective/scores": 17.125, "policy/approxkl_avg": 0.00084353, "policy/clipfrac_avg": 0.0057892, "loss/policy_avg": -0.18964827, "loss/value_avg": 0.51362807, "val/clipfrac_avg": 0.06486779, "policy/entropy_avg": 0.16801262, "val/ratio": 0.98902762, "val/ratio_var": 7.126e-05, "val/num_eos_tokens": 555, "lr": 2.24e-06, "episode": 13920, "epoch": 0.7030303, "global_step/max_steps": "870/1238", "percentage": "70.27%", "elapsed_time": "10h 41m 50s", "remaining_time": "4h 31m 29s"} +{"eps": 0, "objective/kl": 39.21965027, "objective/entropy": -45.07421112, "objective/non_score_reward": -1.96098256, "objective/rlhf_reward": 14.62495422, "objective/scores": 16.625, "policy/approxkl_avg": 0.00091337, "policy/clipfrac_avg": 0.00403097, "loss/policy_avg": 0.02698253, "loss/value_avg": 0.27916098, "val/clipfrac_avg": 0.03648558, "policy/entropy_avg": 0.18420029, "val/ratio": 0.98811066, "val/ratio_var": 5.089e-05, "val/num_eos_tokens": 232, "lr": 2.23e-06, "episode": 13936, "epoch": 0.70383838, "global_step/max_steps": "871/1238", "percentage": "70.36%", "elapsed_time": "10h 42m 21s", "remaining_time": "4h 30m 39s"} +{"eps": 0, "objective/kl": 37.08468628, "objective/entropy": -52.43424988, "objective/non_score_reward": -1.85423446, "objective/rlhf_reward": 14.1848278, "objective/scores": 16.0, "policy/approxkl_avg": 0.00084214, "policy/clipfrac_avg": 0.0026647, "loss/policy_avg": -0.12599462, "loss/value_avg": 0.50077617, "val/clipfrac_avg": 0.00100172, "policy/entropy_avg": 0.17169237, "val/ratio": 0.98810881, "val/ratio_var": 6.611e-05, "val/num_eos_tokens": 274, "lr": 2.22e-06, "episode": 13952, "epoch": 0.70464646, "global_step/max_steps": "872/1238", "percentage": "70.44%", "elapsed_time": "10h 42m 55s", "remaining_time": "4h 29m 51s"} +{"eps": 0, "objective/kl": 41.75783157, "objective/entropy": -138.92434692, "objective/non_score_reward": -2.08789158, "objective/rlhf_reward": 13.09179688, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00084338, "policy/clipfrac_avg": 0.0060525, "loss/policy_avg": 0.35178623, "loss/value_avg": 0.5846591, "val/clipfrac_avg": 0.03221033, "policy/entropy_avg": 0.16450119, "val/ratio": 0.98888373, "val/ratio_var": 5.523e-05, "val/num_eos_tokens": 1144, "lr": 2.21e-06, "episode": 13968, "epoch": 0.70545455, "global_step/max_steps": "873/1238", "percentage": "70.52%", "elapsed_time": "10h 43m 58s", "remaining_time": "4h 29m 14s"} +{"eps": 0, "objective/kl": 37.81830978, "objective/entropy": -122.14696503, "objective/non_score_reward": -1.89091551, "objective/rlhf_reward": 13.29658508, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00066407, "policy/clipfrac_avg": 0.00470848, "loss/policy_avg": -0.00452566, "loss/value_avg": 0.52597708, "val/clipfrac_avg": 0.10728075, "policy/entropy_avg": 0.14041615, "val/ratio": 0.99047798, "val/ratio_var": 6.866e-05, "val/num_eos_tokens": 35, "lr": 2.19e-06, "episode": 13984, "epoch": 0.70626263, "global_step/max_steps": "874/1238", "percentage": "70.60%", "elapsed_time": "10h 44m 45s", "remaining_time": "4h 28m 31s"} +{"eps": 0, "objective/kl": 37.6293869, "objective/entropy": -54.20733261, "objective/non_score_reward": -1.88146925, "objective/rlhf_reward": 14.22009277, "objective/scores": 16.125, "policy/approxkl_avg": 0.00085963, "policy/clipfrac_avg": 0.00539016, "loss/policy_avg": 0.11862925, "loss/value_avg": 1.07227707, "val/clipfrac_avg": 0.05937509, "policy/entropy_avg": 0.17551994, "val/ratio": 0.98782086, "val/ratio_var": 6.485e-05, "val/num_eos_tokens": 201, "lr": 2.18e-06, "episode": 14000, "epoch": 0.70707071, "global_step/max_steps": "875/1238", "percentage": "70.68%", "elapsed_time": "10h 45m 21s", "remaining_time": "4h 27m 43s"} +{"eps": 0, "objective/kl": 44.69777679, "objective/entropy": -16.79524231, "objective/non_score_reward": -2.23488903, "objective/rlhf_reward": 13.14011097, "objective/scores": 15.375, "policy/approxkl_avg": 0.0011914, "policy/clipfrac_avg": 0.00557536, "loss/policy_avg": -0.2030085, "loss/value_avg": 0.55949533, "val/clipfrac_avg": 0.02393021, "policy/entropy_avg": 0.25437927, "val/ratio": 0.98356485, "val/ratio_var": 0.00010361, "val/num_eos_tokens": 24, "lr": 2.17e-06, "episode": 14016, "epoch": 0.70787879, "global_step/max_steps": "876/1238", "percentage": "70.76%", "elapsed_time": "10h 45m 52s", "remaining_time": "4h 26m 54s"} +{"eps": 0, "objective/kl": 50.78942108, "objective/entropy": -105.85357666, "objective/non_score_reward": -2.53947115, "objective/rlhf_reward": 13.07771683, "objective/scores": 15.625, "policy/approxkl_avg": 0.00083869, "policy/clipfrac_avg": 0.00333018, "loss/policy_avg": 0.0506905, "loss/value_avg": 0.64545321, "val/clipfrac_avg": 0.01081987, "policy/entropy_avg": 0.18417263, "val/ratio": 0.98860842, "val/ratio_var": 5.415e-05, "val/num_eos_tokens": 513, "lr": 2.16e-06, "episode": 14032, "epoch": 0.70868687, "global_step/max_steps": "877/1238", "percentage": "70.84%", "elapsed_time": "10h 46m 39s", "remaining_time": "4h 26m 11s"} +{"eps": 0, "objective/kl": 41.63684845, "objective/entropy": -193.71432495, "objective/non_score_reward": -2.08184242, "objective/rlhf_reward": 14.26190758, "objective/scores": 16.375, "policy/approxkl_avg": 0.00070086, "policy/clipfrac_avg": 0.00410365, "loss/policy_avg": -0.32361558, "loss/value_avg": 0.42758209, "val/clipfrac_avg": 7.63e-06, "policy/entropy_avg": 0.14459276, "val/ratio": 0.99034977, "val/ratio_var": 3.226e-05, "val/num_eos_tokens": 323, "lr": 2.15e-06, "episode": 14048, "epoch": 0.70949495, "global_step/max_steps": "878/1238", "percentage": "70.92%", "elapsed_time": "10h 48m 23s", "remaining_time": "4h 25m 51s"} +{"eps": 0, "objective/kl": 34.28295898, "objective/entropy": -45.89549255, "objective/non_score_reward": -1.71414781, "objective/rlhf_reward": 14.69210243, "objective/scores": 16.375, "policy/approxkl_avg": 0.00076509, "policy/clipfrac_avg": 0.00561137, "loss/policy_avg": -0.16454422, "loss/value_avg": 0.77773547, "val/clipfrac_avg": 0.10208054, "policy/entropy_avg": 0.15566254, "val/ratio": 0.98950845, "val/ratio_var": 3.38e-05, "val/num_eos_tokens": 49, "lr": 2.14e-06, "episode": 14064, "epoch": 0.71030303, "global_step/max_steps": "879/1238", "percentage": "71.00%", "elapsed_time": "10h 48m 56s", "remaining_time": "4h 25m 2s"} +{"eps": 0, "objective/kl": 41.30862808, "objective/entropy": -50.96327209, "objective/non_score_reward": -2.06543136, "objective/rlhf_reward": 14.45019341, "objective/scores": 16.5, "policy/approxkl_avg": 0.00093773, "policy/clipfrac_avg": 0.00421791, "loss/policy_avg": -0.02305336, "loss/value_avg": 0.5213716, "val/clipfrac_avg": 0.04448427, "policy/entropy_avg": 0.17507076, "val/ratio": 0.98893499, "val/ratio_var": 3.734e-05, "val/num_eos_tokens": 368, "lr": 2.13e-06, "episode": 14080, "epoch": 0.71111111, "global_step/max_steps": "880/1238", "percentage": "71.08%", "elapsed_time": "10h 49m 36s", "remaining_time": "4h 24m 16s"} +{"eps": 0, "objective/kl": 41.05116653, "objective/entropy": -69.72641754, "objective/non_score_reward": -2.05255842, "objective/rlhf_reward": 14.8302536, "objective/scores": 16.875, "policy/approxkl_avg": 0.00104078, "policy/clipfrac_avg": 0.00337515, "loss/policy_avg": -0.07007279, "loss/value_avg": 0.21621414, "val/clipfrac_avg": 0.03193364, "policy/entropy_avg": 0.20317459, "val/ratio": 0.98610032, "val/ratio_var": 8.361e-05, "val/num_eos_tokens": 551, "lr": 2.12e-06, "episode": 14096, "epoch": 0.71191919, "global_step/max_steps": "881/1238", "percentage": "71.16%", "elapsed_time": "10h 50m 19s", "remaining_time": "4h 23m 31s"} +{"eps": 0, "objective/kl": 45.0096283, "objective/entropy": -86.86426544, "objective/non_score_reward": -2.25048137, "objective/rlhf_reward": 13.91358089, "objective/scores": 16.125, "policy/approxkl_avg": 0.00106061, "policy/clipfrac_avg": 0.00698026, "loss/policy_avg": -0.07847104, "loss/value_avg": 0.26574248, "val/clipfrac_avg": 0.0384758, "policy/entropy_avg": 0.20689774, "val/ratio": 0.98552245, "val/ratio_var": 8.58e-05, "val/num_eos_tokens": 546, "lr": 2.11e-06, "episode": 14112, "epoch": 0.71272727, "global_step/max_steps": "882/1238", "percentage": "71.24%", "elapsed_time": "10h 50m 50s", "remaining_time": "4h 22m 42s"} +{"eps": 0, "objective/kl": 42.61073303, "objective/entropy": -109.19290161, "objective/non_score_reward": -2.13053656, "objective/rlhf_reward": 14.44758797, "objective/scores": 16.625, "policy/approxkl_avg": 0.00103594, "policy/clipfrac_avg": 0.00689279, "loss/policy_avg": -0.13232277, "loss/value_avg": 0.21754739, "val/clipfrac_avg": 0.01065679, "policy/entropy_avg": 0.20017433, "val/ratio": 0.98646891, "val/ratio_var": 4.845e-05, "val/num_eos_tokens": 36, "lr": 2.1e-06, "episode": 14128, "epoch": 0.71353535, "global_step/max_steps": "883/1238", "percentage": "71.32%", "elapsed_time": "10h 51m 32s", "remaining_time": "4h 21m 56s"} +{"eps": 0, "objective/kl": 43.51769638, "objective/entropy": -119.52471924, "objective/non_score_reward": -2.17588472, "objective/rlhf_reward": 14.30067825, "objective/scores": 16.5, "policy/approxkl_avg": 0.00093881, "policy/clipfrac_avg": 0.01213159, "loss/policy_avg": -0.15070915, "loss/value_avg": 0.26654151, "val/clipfrac_avg": 0.00607577, "policy/entropy_avg": 0.18040943, "val/ratio": 0.9873966, "val/ratio_var": 6.622e-05, "val/num_eos_tokens": 620, "lr": 2.08e-06, "episode": 14144, "epoch": 0.71434343, "global_step/max_steps": "884/1238", "percentage": "71.41%", "elapsed_time": "10h 52m 17s", "remaining_time": "4h 21m 12s"} +{"eps": 0, "objective/kl": 45.77116394, "objective/entropy": -62.27338028, "objective/non_score_reward": -2.28855848, "objective/rlhf_reward": 14.61769104, "objective/scores": 16.875, "policy/approxkl_avg": 0.00097187, "policy/clipfrac_avg": 0.00492489, "loss/policy_avg": -0.02853005, "loss/value_avg": 0.43481007, "val/clipfrac_avg": 0.03630942, "policy/entropy_avg": 0.207798, "val/ratio": 0.98708212, "val/ratio_var": 8.348e-05, "val/num_eos_tokens": 179, "lr": 2.07e-06, "episode": 14160, "epoch": 0.71515152, "global_step/max_steps": "885/1238", "percentage": "71.49%", "elapsed_time": "10h 53m 4s", "remaining_time": "4h 20m 29s"} +{"eps": 0, "objective/kl": 32.41828918, "objective/entropy": -84.31056213, "objective/non_score_reward": -1.62091458, "objective/rlhf_reward": 15.33221054, "objective/scores": 17.0, "policy/approxkl_avg": 0.00085894, "policy/clipfrac_avg": 0.00275231, "loss/policy_avg": -0.15033548, "loss/value_avg": 0.60432488, "val/clipfrac_avg": 0.03947105, "policy/entropy_avg": 0.18497181, "val/ratio": 0.98793137, "val/ratio_var": 5.601e-05, "val/num_eos_tokens": 180, "lr": 2.06e-06, "episode": 14176, "epoch": 0.7159596, "global_step/max_steps": "886/1238", "percentage": "71.57%", "elapsed_time": "10h 53m 37s", "remaining_time": "4h 19m 40s"} +{"eps": 0, "objective/kl": 36.84368134, "objective/entropy": -51.08298492, "objective/non_score_reward": -1.84218407, "objective/rlhf_reward": 14.08750343, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0010462, "policy/clipfrac_avg": 0.00458747, "loss/policy_avg": 0.22797719, "loss/value_avg": 1.20743418, "val/clipfrac_avg": 0.01551971, "policy/entropy_avg": 0.2259798, "val/ratio": 0.98491859, "val/ratio_var": 6.349e-05, "val/num_eos_tokens": 448, "lr": 2.05e-06, "episode": 14192, "epoch": 0.71676768, "global_step/max_steps": "887/1238", "percentage": "71.65%", "elapsed_time": "10h 54m 11s", "remaining_time": "4h 18m 52s"} +{"eps": 0, "objective/kl": 42.72069931, "objective/entropy": -154.47012329, "objective/non_score_reward": -2.13603497, "objective/rlhf_reward": 14.95771503, "objective/scores": 17.125, "policy/approxkl_avg": 0.00092994, "policy/clipfrac_avg": 0.00832129, "loss/policy_avg": -0.02626729, "loss/value_avg": 0.59659278, "val/clipfrac_avg": 0.01875, "policy/entropy_avg": 0.17518932, "val/ratio": 0.9883604, "val/ratio_var": 7.326e-05, "val/num_eos_tokens": 428, "lr": 2.04e-06, "episode": 14208, "epoch": 0.71757576, "global_step/max_steps": "888/1238", "percentage": "71.73%", "elapsed_time": "10h 55m 34s", "remaining_time": "4h 18m 23s"} +{"eps": 0, "objective/kl": 33.89815521, "objective/entropy": -128.48831177, "objective/non_score_reward": -1.6949079, "objective/rlhf_reward": 12.59806061, "objective/scores": 14.3125, "policy/approxkl_avg": 0.00067056, "policy/clipfrac_avg": 0.00583194, "loss/policy_avg": 0.40078703, "loss/value_avg": 1.15348542, "val/clipfrac_avg": 0.02483724, "policy/entropy_avg": 0.21662331, "val/ratio": 0.99128854, "val/ratio_var": 7.058e-05, "val/num_eos_tokens": 47, "lr": 2.03e-06, "episode": 14224, "epoch": 0.71838384, "global_step/max_steps": "889/1238", "percentage": "71.81%", "elapsed_time": "10h 56m 28s", "remaining_time": "4h 17m 43s"} +{"eps": 0, "objective/kl": 40.06652069, "objective/entropy": -86.98061371, "objective/non_score_reward": -2.00332594, "objective/rlhf_reward": 14.03573704, "objective/scores": 16.0, "policy/approxkl_avg": 0.00088156, "policy/clipfrac_avg": 0.00601772, "loss/policy_avg": -0.08559307, "loss/value_avg": 0.32921553, "val/clipfrac_avg": 0.06256754, "policy/entropy_avg": 0.1755352, "val/ratio": 0.98729843, "val/ratio_var": 8.527e-05, "val/num_eos_tokens": 20, "lr": 2.02e-06, "episode": 14240, "epoch": 0.71919192, "global_step/max_steps": "890/1238", "percentage": "71.89%", "elapsed_time": "10h 57m 4s", "remaining_time": "4h 16m 55s"} +{"eps": 0, "objective/kl": 41.95650864, "objective/entropy": -99.16207123, "objective/non_score_reward": -2.09782529, "objective/rlhf_reward": 13.745924, "objective/scores": 15.875, "policy/approxkl_avg": 0.00078978, "policy/clipfrac_avg": 0.00430315, "loss/policy_avg": -0.17781906, "loss/value_avg": 0.22757252, "val/clipfrac_avg": 0.01233413, "policy/entropy_avg": 0.15855598, "val/ratio": 0.98948979, "val/ratio_var": 5.431e-05, "val/num_eos_tokens": 424, "lr": 2.01e-06, "episode": 14256, "epoch": 0.72, "global_step/max_steps": "891/1238", "percentage": "71.97%", "elapsed_time": "10h 57m 42s", "remaining_time": "4h 16m 8s"} +{"eps": 0, "objective/kl": 37.07469177, "objective/entropy": -61.42802048, "objective/non_score_reward": -1.85373473, "objective/rlhf_reward": 13.04470253, "objective/scores": 14.875, "policy/approxkl_avg": 0.00084864, "policy/clipfrac_avg": 0.0067578, "loss/policy_avg": -0.36396489, "loss/value_avg": 0.86622345, "val/clipfrac_avg": 0.05480193, "policy/entropy_avg": 0.18663216, "val/ratio": 0.98832607, "val/ratio_var": 6.73e-05, "val/num_eos_tokens": 584, "lr": 2e-06, "episode": 14272, "epoch": 0.72080808, "global_step/max_steps": "892/1238", "percentage": "72.05%", "elapsed_time": "10h 58m 20s", "remaining_time": "4h 15m 21s"} +{"eps": 0, "objective/kl": 42.03372574, "objective/entropy": -58.38874054, "objective/non_score_reward": -2.10168624, "objective/rlhf_reward": 12.82800102, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00080784, "policy/clipfrac_avg": 0.00471095, "loss/policy_avg": -0.1217151, "loss/value_avg": 0.34999394, "val/clipfrac_avg": 0.03385914, "policy/entropy_avg": 0.16219139, "val/ratio": 0.99099189, "val/ratio_var": 4.974e-05, "val/num_eos_tokens": 211, "lr": 1.99e-06, "episode": 14288, "epoch": 0.72161616, "global_step/max_steps": "893/1238", "percentage": "72.13%", "elapsed_time": "10h 58m 52s", "remaining_time": "4h 14m 33s"} +{"eps": 0, "objective/kl": 43.86260986, "objective/entropy": -72.03929138, "objective/non_score_reward": -2.19313049, "objective/rlhf_reward": 13.32249451, "objective/scores": 15.5, "policy/approxkl_avg": 0.00098598, "policy/clipfrac_avg": 0.00486141, "loss/policy_avg": 0.15615974, "loss/value_avg": 0.49859789, "val/clipfrac_avg": 0.00886926, "policy/entropy_avg": 0.1980648, "val/ratio": 0.98696733, "val/ratio_var": 5.953e-05, "val/num_eos_tokens": 389, "lr": 1.98e-06, "episode": 14304, "epoch": 0.72242424, "global_step/max_steps": "894/1238", "percentage": "72.21%", "elapsed_time": "10h 59m 27s", "remaining_time": "4h 13m 45s"} +{"eps": 0, "objective/kl": 38.79567719, "objective/entropy": -30.88048744, "objective/non_score_reward": -1.93978381, "objective/rlhf_reward": 12.95865345, "objective/scores": 14.875, "policy/approxkl_avg": 0.00110678, "policy/clipfrac_avg": 0.00475399, "loss/policy_avg": -0.01865309, "loss/value_avg": 0.32360113, "val/clipfrac_avg": 0.01328514, "policy/entropy_avg": 0.21401215, "val/ratio": 0.98442125, "val/ratio_var": 5.579e-05, "val/num_eos_tokens": 190, "lr": 1.97e-06, "episode": 14320, "epoch": 0.72323232, "global_step/max_steps": "895/1238", "percentage": "72.29%", "elapsed_time": "11h 0m 0s", "remaining_time": "4h 12m 56s"} +{"eps": 0, "objective/kl": 40.95935059, "objective/entropy": -88.01570129, "objective/non_score_reward": -2.04796743, "objective/rlhf_reward": 13.35046959, "objective/scores": 15.375, "policy/approxkl_avg": 0.00103093, "policy/clipfrac_avg": 0.00548341, "loss/policy_avg": -0.04005177, "loss/value_avg": 0.56448841, "val/clipfrac_avg": 0.01046415, "policy/entropy_avg": 0.21037865, "val/ratio": 0.98658079, "val/ratio_var": 9.776e-05, "val/num_eos_tokens": 539, "lr": 1.96e-06, "episode": 14336, "epoch": 0.7240404, "global_step/max_steps": "896/1238", "percentage": "72.37%", "elapsed_time": "11h 0m 34s", "remaining_time": "4h 12m 8s"} +{"eps": 0, "objective/kl": 37.47423553, "objective/entropy": -98.20949554, "objective/non_score_reward": -1.87371182, "objective/rlhf_reward": 13.63410091, "objective/scores": 15.5, "policy/approxkl_avg": 0.00079853, "policy/clipfrac_avg": 0.00506318, "loss/policy_avg": -0.13584074, "loss/value_avg": 0.53688991, "val/clipfrac_avg": 0.06062069, "policy/entropy_avg": 0.1458236, "val/ratio": 0.99035573, "val/ratio_var": 5.068e-05, "val/num_eos_tokens": 1182, "lr": 1.95e-06, "episode": 14352, "epoch": 0.72484848, "global_step/max_steps": "897/1238", "percentage": "72.46%", "elapsed_time": "11h 1m 36s", "remaining_time": "4h 11m 30s"} +{"eps": 0, "objective/kl": 41.13695526, "objective/entropy": -110.30941772, "objective/non_score_reward": -2.05684781, "objective/rlhf_reward": 13.72440243, "objective/scores": 15.75, "policy/approxkl_avg": 0.00116264, "policy/clipfrac_avg": 0.00675421, "loss/policy_avg": 0.01142725, "loss/value_avg": 0.26316276, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.22265816, "val/ratio": 0.9850179, "val/ratio_var": 8.877e-05, "val/num_eos_tokens": 10, "lr": 1.94e-06, "episode": 14368, "epoch": 0.72565657, "global_step/max_steps": "898/1238", "percentage": "72.54%", "elapsed_time": "11h 2m 18s", "remaining_time": "4h 10m 45s"} +{"eps": 0, "objective/kl": 48.55796051, "objective/entropy": -101.70489502, "objective/non_score_reward": -2.42789793, "objective/rlhf_reward": 13.31428909, "objective/scores": 15.75, "policy/approxkl_avg": 0.0009827, "policy/clipfrac_avg": 0.00461012, "loss/policy_avg": 0.11541194, "loss/value_avg": 0.46310914, "val/clipfrac_avg": 0.01362092, "policy/entropy_avg": 0.17246819, "val/ratio": 0.98769736, "val/ratio_var": 5.493e-05, "val/num_eos_tokens": 516, "lr": 1.92e-06, "episode": 14384, "epoch": 0.72646465, "global_step/max_steps": "899/1238", "percentage": "72.62%", "elapsed_time": "11h 3m 2s", "remaining_time": "4h 10m 1s"} +{"eps": 0, "objective/kl": 41.26939011, "objective/entropy": -91.96019745, "objective/non_score_reward": -2.06346941, "objective/rlhf_reward": 12.52246761, "objective/scores": 14.5625, "policy/approxkl_avg": 0.0009332, "policy/clipfrac_avg": 0.00430764, "loss/policy_avg": 0.079368, "loss/value_avg": 0.66165167, "val/clipfrac_avg": 0.01476163, "policy/entropy_avg": 0.18123245, "val/ratio": 0.98759735, "val/ratio_var": 7.476e-05, "val/num_eos_tokens": 595, "lr": 1.91e-06, "episode": 14400, "epoch": 0.72727273, "global_step/max_steps": "900/1238", "percentage": "72.70%", "elapsed_time": "11h 3m 37s", "remaining_time": "4h 9m 13s"} +{"eps": 0, "objective/kl": 39.73054123, "objective/entropy": -59.23660278, "objective/non_score_reward": -1.98652709, "objective/rlhf_reward": 12.43534851, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00117044, "policy/clipfrac_avg": 0.00493789, "loss/policy_avg": -0.02671998, "loss/value_avg": 0.71748352, "val/clipfrac_avg": 0.1326164, "policy/entropy_avg": 0.22887516, "val/ratio": 0.98483729, "val/ratio_var": 8.884e-05, "val/num_eos_tokens": 121, "lr": 1.9e-06, "episode": 14416, "epoch": 0.72808081, "global_step/max_steps": "901/1238", "percentage": "72.78%", "elapsed_time": "11h 4m 44s", "remaining_time": "4h 8m 37s"} +{"eps": 0, "objective/kl": 46.26803589, "objective/entropy": -53.46650696, "objective/non_score_reward": -2.31340194, "objective/rlhf_reward": 14.38972282, "objective/scores": 16.75, "policy/approxkl_avg": 0.00100223, "policy/clipfrac_avg": 0.00406684, "loss/policy_avg": 0.05788305, "loss/value_avg": 0.25914964, "val/clipfrac_avg": 0.01396515, "policy/entropy_avg": 0.20316124, "val/ratio": 0.98662806, "val/ratio_var": 5.23e-05, "val/num_eos_tokens": 455, "lr": 1.89e-06, "episode": 14432, "epoch": 0.72888889, "global_step/max_steps": "902/1238", "percentage": "72.86%", "elapsed_time": "11h 5m 38s", "remaining_time": "4h 7m 57s"} +{"eps": 0, "objective/kl": 46.20999146, "objective/entropy": -65.68837738, "objective/non_score_reward": -2.31049967, "objective/rlhf_reward": 16.78325081, "objective/scores": 19.125, "policy/approxkl_avg": 0.00100661, "policy/clipfrac_avg": 0.00540682, "loss/policy_avg": -0.11720985, "loss/value_avg": 0.23323998, "val/clipfrac_avg": 0.01505159, "policy/entropy_avg": 0.20101547, "val/ratio": 0.98652995, "val/ratio_var": 5.016e-05, "val/num_eos_tokens": 205, "lr": 1.88e-06, "episode": 14448, "epoch": 0.72969697, "global_step/max_steps": "903/1238", "percentage": "72.94%", "elapsed_time": "11h 6m 11s", "remaining_time": "4h 7m 8s"} +{"eps": 0, "objective/kl": 37.29925537, "objective/entropy": -118.07336426, "objective/non_score_reward": -1.86496282, "objective/rlhf_reward": 15.88503742, "objective/scores": 17.75, "policy/approxkl_avg": 0.00075889, "policy/clipfrac_avg": 0.00239067, "loss/policy_avg": -0.4681462, "loss/value_avg": 0.80522239, "val/clipfrac_avg": 0.13810205, "policy/entropy_avg": 0.15973949, "val/ratio": 0.99021357, "val/ratio_var": 7.257e-05, "val/num_eos_tokens": 155, "lr": 1.87e-06, "episode": 14464, "epoch": 0.73050505, "global_step/max_steps": "904/1238", "percentage": "73.02%", "elapsed_time": "11h 6m 43s", "remaining_time": "4h 6m 20s"} +{"eps": 0, "objective/kl": 39.16048431, "objective/entropy": -22.49644089, "objective/non_score_reward": -1.95802426, "objective/rlhf_reward": 12.32322598, "objective/scores": 14.25, "policy/approxkl_avg": 0.00107185, "policy/clipfrac_avg": 0.0042765, "loss/policy_avg": 0.1916002, "loss/value_avg": 0.90718901, "val/clipfrac_avg": 0.03053325, "policy/entropy_avg": 0.19223785, "val/ratio": 0.98710907, "val/ratio_var": 4.572e-05, "val/num_eos_tokens": 240, "lr": 1.86e-06, "episode": 14480, "epoch": 0.73131313, "global_step/max_steps": "905/1238", "percentage": "73.10%", "elapsed_time": "11h 7m 20s", "remaining_time": "4h 5m 32s"} +{"eps": 0, "objective/kl": 30.63116264, "objective/entropy": -52.41043472, "objective/non_score_reward": -1.53155828, "objective/rlhf_reward": 16.29656601, "objective/scores": 17.875, "policy/approxkl_avg": 0.00110776, "policy/clipfrac_avg": 0.00652417, "loss/policy_avg": -0.11877916, "loss/value_avg": 0.68734223, "val/clipfrac_avg": 0.00029481, "policy/entropy_avg": 0.20643425, "val/ratio": 0.98613894, "val/ratio_var": 6.196e-05, "val/num_eos_tokens": 261, "lr": 1.85e-06, "episode": 14496, "epoch": 0.73212121, "global_step/max_steps": "906/1238", "percentage": "73.18%", "elapsed_time": "11h 7m 57s", "remaining_time": "4h 4m 46s"} +{"eps": 0, "objective/kl": 43.33234406, "objective/entropy": -61.85654449, "objective/non_score_reward": -2.16661739, "objective/rlhf_reward": 13.14588261, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00105279, "policy/clipfrac_avg": 0.00457506, "loss/policy_avg": 0.10565789, "loss/value_avg": 0.50670052, "val/clipfrac_avg": 0.00558036, "policy/entropy_avg": 0.18899155, "val/ratio": 0.98727882, "val/ratio_var": 7.066e-05, "val/num_eos_tokens": 496, "lr": 1.84e-06, "episode": 14512, "epoch": 0.73292929, "global_step/max_steps": "907/1238", "percentage": "73.26%", "elapsed_time": "11h 8m 28s", "remaining_time": "4h 3m 57s"} +{"eps": 0, "objective/kl": 37.61989594, "objective/entropy": -91.67915344, "objective/non_score_reward": -1.8809948, "objective/rlhf_reward": 14.2205677, "objective/scores": 16.125, "policy/approxkl_avg": 0.00094947, "policy/clipfrac_avg": 0.00709912, "loss/policy_avg": 0.04191252, "loss/value_avg": 0.30707109, "val/clipfrac_avg": 0.07242873, "policy/entropy_avg": 0.17319107, "val/ratio": 0.98837966, "val/ratio_var": 7.367e-05, "val/num_eos_tokens": 437, "lr": 1.83e-06, "episode": 14528, "epoch": 0.73373737, "global_step/max_steps": "908/1238", "percentage": "73.34%", "elapsed_time": "11h 9m 15s", "remaining_time": "4h 3m 14s"} +{"eps": 0, "objective/kl": 36.91444397, "objective/entropy": -92.03844452, "objective/non_score_reward": -1.8457222, "objective/rlhf_reward": 13.8027153, "objective/scores": 15.625, "policy/approxkl_avg": 0.0009771, "policy/clipfrac_avg": 0.00556906, "loss/policy_avg": 0.15317559, "loss/value_avg": 0.73794425, "val/clipfrac_avg": 0.0405195, "policy/entropy_avg": 0.18958092, "val/ratio": 0.98892319, "val/ratio_var": 6.142e-05, "val/num_eos_tokens": 56, "lr": 1.82e-06, "episode": 14544, "epoch": 0.73454545, "global_step/max_steps": "909/1238", "percentage": "73.42%", "elapsed_time": "11h 10m 24s", "remaining_time": "4h 2m 38s"} +{"eps": 0, "objective/kl": 38.83802032, "objective/entropy": -111.80148315, "objective/non_score_reward": -1.94190121, "objective/rlhf_reward": 14.40184879, "objective/scores": 16.375, "policy/approxkl_avg": 0.000904, "policy/clipfrac_avg": 0.00531613, "loss/policy_avg": 0.16024318, "loss/value_avg": 0.30254692, "val/clipfrac_avg": 0.00018382, "policy/entropy_avg": 0.17273903, "val/ratio": 0.98911309, "val/ratio_var": 7.235e-05, "val/num_eos_tokens": 342, "lr": 1.81e-06, "episode": 14560, "epoch": 0.73535354, "global_step/max_steps": "910/1238", "percentage": "73.51%", "elapsed_time": "11h 10m 56s", "remaining_time": "4h 1m 50s"} +{"eps": 0, "objective/kl": 44.55670166, "objective/entropy": -98.02986145, "objective/non_score_reward": -2.22783494, "objective/rlhf_reward": 13.0924778, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00093366, "policy/clipfrac_avg": 0.00482254, "loss/policy_avg": 0.09882548, "loss/value_avg": 0.61213279, "val/clipfrac_avg": 0.00507727, "policy/entropy_avg": 0.1792078, "val/ratio": 0.98752236, "val/ratio_var": 6.38e-05, "val/num_eos_tokens": 426, "lr": 1.8e-06, "episode": 14576, "epoch": 0.73616162, "global_step/max_steps": "911/1238", "percentage": "73.59%", "elapsed_time": "11h 11m 32s", "remaining_time": "4h 1m 2s"} +{"eps": 0, "objective/kl": 36.0546608, "objective/entropy": -57.16059875, "objective/non_score_reward": -1.80273306, "objective/rlhf_reward": 13.11914253, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00096705, "policy/clipfrac_avg": 0.00586616, "loss/policy_avg": -0.15951847, "loss/value_avg": 0.56488454, "val/clipfrac_avg": 0.00082237, "policy/entropy_avg": 0.17210197, "val/ratio": 0.9892112, "val/ratio_var": 4.959e-05, "val/num_eos_tokens": 45, "lr": 1.79e-06, "episode": 14592, "epoch": 0.7369697, "global_step/max_steps": "912/1238", "percentage": "73.67%", "elapsed_time": "11h 12m 5s", "remaining_time": "4h 0m 14s"} +{"eps": 0, "objective/kl": 46.18011475, "objective/entropy": -146.66589355, "objective/non_score_reward": -2.30900574, "objective/rlhf_reward": 14.51130676, "objective/scores": 16.875, "policy/approxkl_avg": 0.00093072, "policy/clipfrac_avg": 0.0061439, "loss/policy_avg": 0.19401941, "loss/value_avg": 0.38150606, "val/clipfrac_avg": 0.01809039, "policy/entropy_avg": 0.18556425, "val/ratio": 0.98741895, "val/ratio_var": 0.00011523, "val/num_eos_tokens": 698, "lr": 1.78e-06, "episode": 14608, "epoch": 0.73777778, "global_step/max_steps": "913/1238", "percentage": "73.75%", "elapsed_time": "11h 12m 56s", "remaining_time": "3h 59m 32s"} +{"eps": 0, "objective/kl": 35.78252411, "objective/entropy": -84.84890747, "objective/non_score_reward": -1.7891264, "objective/rlhf_reward": 13.2655611, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00093215, "policy/clipfrac_avg": 0.00500773, "loss/policy_avg": -0.02095976, "loss/value_avg": 0.72243726, "val/clipfrac_avg": 0.02376894, "policy/entropy_avg": 0.18702126, "val/ratio": 0.98742843, "val/ratio_var": 8.1e-05, "val/num_eos_tokens": 403, "lr": 1.77e-06, "episode": 14624, "epoch": 0.73858586, "global_step/max_steps": "914/1238", "percentage": "73.83%", "elapsed_time": "11h 13m 31s", "remaining_time": "3h 58m 45s"} +{"eps": 0, "objective/kl": 38.50158691, "objective/entropy": -63.22113037, "objective/non_score_reward": -1.92507935, "objective/rlhf_reward": 13.91867065, "objective/scores": 15.875, "policy/approxkl_avg": 0.00085014, "policy/clipfrac_avg": 0.0047064, "loss/policy_avg": -0.31347317, "loss/value_avg": 0.57656211, "val/clipfrac_avg": 0.06931676, "policy/entropy_avg": 0.16005135, "val/ratio": 0.98829925, "val/ratio_var": 6.203e-05, "val/num_eos_tokens": 171, "lr": 1.76e-06, "episode": 14640, "epoch": 0.73939394, "global_step/max_steps": "915/1238", "percentage": "73.91%", "elapsed_time": "11h 14m 19s", "remaining_time": "3h 58m 2s"} +{"eps": 0, "objective/kl": 42.95529175, "objective/entropy": -43.27140045, "objective/non_score_reward": -2.14776468, "objective/rlhf_reward": 13.87567329, "objective/scores": 16.0, "policy/approxkl_avg": 0.00094958, "policy/clipfrac_avg": 0.00558306, "loss/policy_avg": -0.2850199, "loss/value_avg": 0.86794376, "val/clipfrac_avg": 0.06379853, "policy/entropy_avg": 0.17831039, "val/ratio": 0.98767984, "val/ratio_var": 4.003e-05, "val/num_eos_tokens": 103, "lr": 1.75e-06, "episode": 14656, "epoch": 0.74020202, "global_step/max_steps": "916/1238", "percentage": "73.99%", "elapsed_time": "11h 14m 59s", "remaining_time": "3h 57m 16s"} +{"eps": 0, "objective/kl": 44.6399231, "objective/entropy": -79.67193604, "objective/non_score_reward": -2.2319963, "objective/rlhf_reward": 13.46331596, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00089415, "policy/clipfrac_avg": 0.00667461, "loss/policy_avg": 0.21622401, "loss/value_avg": 0.22584835, "val/clipfrac_avg": 0.0295848, "policy/entropy_avg": 0.17199516, "val/ratio": 0.98860198, "val/ratio_var": 4.661e-05, "val/num_eos_tokens": 50, "lr": 1.74e-06, "episode": 14672, "epoch": 0.7410101, "global_step/max_steps": "917/1238", "percentage": "74.07%", "elapsed_time": "11h 15m 46s", "remaining_time": "3h 56m 33s"} +{"eps": 0, "objective/kl": 38.6593132, "objective/entropy": -68.00552368, "objective/non_score_reward": -1.93296552, "objective/rlhf_reward": 13.88734627, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00082512, "policy/clipfrac_avg": 0.00550935, "loss/policy_avg": -0.00064861, "loss/value_avg": 0.29627186, "val/clipfrac_avg": 0.00382658, "policy/entropy_avg": 0.16430855, "val/ratio": 0.98889709, "val/ratio_var": 3.051e-05, "val/num_eos_tokens": 44, "lr": 1.73e-06, "episode": 14688, "epoch": 0.74181818, "global_step/max_steps": "918/1238", "percentage": "74.15%", "elapsed_time": "11h 16m 25s", "remaining_time": "3h 55m 47s"} +{"eps": 0, "objective/kl": 40.42371368, "objective/entropy": -93.2693634, "objective/non_score_reward": -2.02118587, "objective/rlhf_reward": 15.16631413, "objective/scores": 17.25, "policy/approxkl_avg": 0.00097431, "policy/clipfrac_avg": 0.00289373, "loss/policy_avg": -0.34626719, "loss/value_avg": 0.71529019, "val/clipfrac_avg": 0.04902752, "policy/entropy_avg": 0.18361568, "val/ratio": 0.98775071, "val/ratio_var": 5.375e-05, "val/num_eos_tokens": 136, "lr": 1.72e-06, "episode": 14704, "epoch": 0.74262626, "global_step/max_steps": "919/1238", "percentage": "74.23%", "elapsed_time": "11h 17m 16s", "remaining_time": "3h 55m 5s"} +{"eps": 0, "objective/kl": 36.17670441, "objective/entropy": -81.48149109, "objective/non_score_reward": -1.80883527, "objective/rlhf_reward": 12.58960152, "objective/scores": 14.375, "policy/approxkl_avg": 0.00084957, "policy/clipfrac_avg": 0.00380708, "loss/policy_avg": 0.12743002, "loss/value_avg": 1.23090529, "val/clipfrac_avg": 0.00742737, "policy/entropy_avg": 0.16942787, "val/ratio": 0.99020922, "val/ratio_var": 5.267e-05, "val/num_eos_tokens": 270, "lr": 1.71e-06, "episode": 14720, "epoch": 0.74343434, "global_step/max_steps": "920/1238", "percentage": "74.31%", "elapsed_time": "11h 17m 48s", "remaining_time": "3h 54m 17s"} +{"eps": 0, "objective/kl": 42.41930771, "objective/entropy": -74.04690552, "objective/non_score_reward": -2.12096548, "objective/rlhf_reward": 13.51184654, "objective/scores": 15.625, "policy/approxkl_avg": 0.00090733, "policy/clipfrac_avg": 0.00485712, "loss/policy_avg": -0.25786677, "loss/value_avg": 0.59427345, "val/clipfrac_avg": 0.04321744, "policy/entropy_avg": 0.18361282, "val/ratio": 0.98777384, "val/ratio_var": 6.433e-05, "val/num_eos_tokens": 56, "lr": 1.7e-06, "episode": 14736, "epoch": 0.74424242, "global_step/max_steps": "921/1238", "percentage": "74.39%", "elapsed_time": "11h 18m 28s", "remaining_time": "3h 53m 31s"} +{"eps": 0, "objective/kl": 35.72700882, "objective/entropy": -54.97579193, "objective/non_score_reward": -1.78635049, "objective/rlhf_reward": 13.39333725, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00101834, "policy/clipfrac_avg": 0.0037684, "loss/policy_avg": 0.12189103, "loss/value_avg": 0.41360408, "val/clipfrac_avg": 0.00086102, "policy/entropy_avg": 0.18449402, "val/ratio": 0.98727381, "val/ratio_var": 3.092e-05, "val/num_eos_tokens": 362, "lr": 1.69e-06, "episode": 14752, "epoch": 0.74505051, "global_step/max_steps": "922/1238", "percentage": "74.47%", "elapsed_time": "11h 19m 10s", "remaining_time": "3h 52m 46s"} +{"eps": 0, "objective/kl": 40.59729767, "objective/entropy": -15.74696255, "objective/non_score_reward": -2.02986503, "objective/rlhf_reward": 12.38810349, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00107701, "policy/clipfrac_avg": 0.00541363, "loss/policy_avg": -0.07059651, "loss/value_avg": 0.64639664, "val/clipfrac_avg": 0.00856552, "policy/entropy_avg": 0.21207809, "val/ratio": 0.98612684, "val/ratio_var": 5.635e-05, "val/num_eos_tokens": 175, "lr": 1.68e-06, "episode": 14768, "epoch": 0.74585859, "global_step/max_steps": "923/1238", "percentage": "74.56%", "elapsed_time": "11h 19m 42s", "remaining_time": "3h 51m 58s"} +{"eps": 0, "objective/kl": 34.25519943, "objective/entropy": -118.91422272, "objective/non_score_reward": -1.71275997, "objective/rlhf_reward": 12.22473907, "objective/scores": 13.9375, "policy/approxkl_avg": 0.00070982, "policy/clipfrac_avg": 0.0022263, "loss/policy_avg": 0.06915352, "loss/value_avg": 1.46515667, "val/clipfrac_avg": 0.06050519, "policy/entropy_avg": 0.13862562, "val/ratio": 0.99124217, "val/ratio_var": 4.531e-05, "val/num_eos_tokens": 270, "lr": 1.67e-06, "episode": 14784, "epoch": 0.74666667, "global_step/max_steps": "924/1238", "percentage": "74.64%", "elapsed_time": "11h 20m 21s", "remaining_time": "3h 51m 12s"} +{"eps": 0, "objective/kl": 38.9585495, "objective/entropy": -81.50097656, "objective/non_score_reward": -1.94792747, "objective/rlhf_reward": 15.45051003, "objective/scores": 17.375, "policy/approxkl_avg": 0.00092775, "policy/clipfrac_avg": 0.00495231, "loss/policy_avg": 0.18248242, "loss/value_avg": 1.6025517, "val/clipfrac_avg": 0.01658271, "policy/entropy_avg": 0.16534215, "val/ratio": 0.98833001, "val/ratio_var": 5.89e-05, "val/num_eos_tokens": 323, "lr": 1.66e-06, "episode": 14800, "epoch": 0.74747475, "global_step/max_steps": "925/1238", "percentage": "74.72%", "elapsed_time": "11h 20m 59s", "remaining_time": "3h 50m 25s"} +{"eps": 0, "objective/kl": 35.18474579, "objective/entropy": -46.54087067, "objective/non_score_reward": -1.75923753, "objective/rlhf_reward": 13.22513771, "objective/scores": 15.0, "policy/approxkl_avg": 0.0009728, "policy/clipfrac_avg": 0.00237955, "loss/policy_avg": -0.16795751, "loss/value_avg": 0.39929, "val/clipfrac_avg": 0.04448857, "policy/entropy_avg": 0.21690035, "val/ratio": 0.98690617, "val/ratio_var": 6.481e-05, "val/num_eos_tokens": 163, "lr": 1.65e-06, "episode": 14816, "epoch": 0.74828283, "global_step/max_steps": "926/1238", "percentage": "74.80%", "elapsed_time": "11h 21m 35s", "remaining_time": "3h 49m 39s"} +{"eps": 0, "objective/kl": 33.04714203, "objective/entropy": -73.27378845, "objective/non_score_reward": -1.65235734, "objective/rlhf_reward": 13.3320179, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008206, "policy/clipfrac_avg": 0.0039194, "loss/policy_avg": -0.31866944, "loss/value_avg": 0.93309009, "val/clipfrac_avg": 0.07463381, "policy/entropy_avg": 0.1740458, "val/ratio": 0.98909736, "val/ratio_var": 8.333e-05, "val/num_eos_tokens": 165, "lr": 1.64e-06, "episode": 14832, "epoch": 0.74909091, "global_step/max_steps": "927/1238", "percentage": "74.88%", "elapsed_time": "11h 22m 7s", "remaining_time": "3h 48m 50s"} +{"eps": 0, "objective/kl": 41.20008469, "objective/entropy": -108.95191193, "objective/non_score_reward": -2.06000447, "objective/rlhf_reward": 14.13530827, "objective/scores": 16.25, "policy/approxkl_avg": 0.00073909, "policy/clipfrac_avg": 0.00218007, "loss/policy_avg": -0.01865103, "loss/value_avg": 0.23730481, "val/clipfrac_avg": 0.02866453, "policy/entropy_avg": 0.15365887, "val/ratio": 0.98992097, "val/ratio_var": 5.118e-05, "val/num_eos_tokens": 497, "lr": 1.63e-06, "episode": 14848, "epoch": 0.74989899, "global_step/max_steps": "928/1238", "percentage": "74.96%", "elapsed_time": "11h 22m 56s", "remaining_time": "3h 48m 8s"} +{"eps": 0, "objective/kl": 47.88815689, "objective/entropy": -53.78815842, "objective/non_score_reward": -2.39440799, "objective/rlhf_reward": 12.57434273, "objective/scores": 15.0, "policy/approxkl_avg": 0.00089382, "policy/clipfrac_avg": 0.0031494, "loss/policy_avg": 0.11356083, "loss/value_avg": 0.50692266, "val/clipfrac_avg": 0.01729911, "policy/entropy_avg": 0.18184018, "val/ratio": 0.98890126, "val/ratio_var": 3.912e-05, "val/num_eos_tokens": 19, "lr": 1.62e-06, "episode": 14864, "epoch": 0.75070707, "global_step/max_steps": "929/1238", "percentage": "75.04%", "elapsed_time": "11h 23m 31s", "remaining_time": "3h 47m 21s"} +{"eps": 0, "objective/kl": 37.49846649, "objective/entropy": -104.18078613, "objective/non_score_reward": -1.87492347, "objective/rlhf_reward": 15.95320129, "objective/scores": 17.875, "policy/approxkl_avg": 0.00091158, "policy/clipfrac_avg": 0.00545195, "loss/policy_avg": -0.07865976, "loss/value_avg": 0.41433531, "val/clipfrac_avg": 0.05804021, "policy/entropy_avg": 0.17452908, "val/ratio": 0.98873311, "val/ratio_var": 6.706e-05, "val/num_eos_tokens": 27, "lr": 1.61e-06, "episode": 14880, "epoch": 0.75151515, "global_step/max_steps": "930/1238", "percentage": "75.12%", "elapsed_time": "11h 24m 3s", "remaining_time": "3h 46m 32s"} +{"eps": 0, "objective/kl": 42.68102264, "objective/entropy": -97.50758362, "objective/non_score_reward": -2.13405108, "objective/rlhf_reward": 13.49876213, "objective/scores": 15.625, "policy/approxkl_avg": 0.00080757, "policy/clipfrac_avg": 0.0113112, "loss/policy_avg": -0.03540248, "loss/value_avg": 0.53589714, "val/clipfrac_avg": 0.00388505, "policy/entropy_avg": 0.1630106, "val/ratio": 0.98903096, "val/ratio_var": 5.21e-05, "val/num_eos_tokens": 302, "lr": 1.6e-06, "episode": 14896, "epoch": 0.75232323, "global_step/max_steps": "931/1238", "percentage": "75.20%", "elapsed_time": "11h 24m 33s", "remaining_time": "3h 45m 44s"} +{"eps": 0, "objective/kl": 32.46774292, "objective/entropy": -78.21574402, "objective/non_score_reward": -1.6233871, "objective/rlhf_reward": 14.37270641, "objective/scores": 16.0, "policy/approxkl_avg": 0.00086478, "policy/clipfrac_avg": 0.00565621, "loss/policy_avg": -0.30045462, "loss/value_avg": 0.87271357, "val/clipfrac_avg": 0.00159196, "policy/entropy_avg": 0.17721939, "val/ratio": 0.98857665, "val/ratio_var": 6.425e-05, "val/num_eos_tokens": 12, "lr": 1.59e-06, "episode": 14912, "epoch": 0.75313131, "global_step/max_steps": "932/1238", "percentage": "75.28%", "elapsed_time": "11h 25m 0s", "remaining_time": "3h 44m 54s"} +{"eps": 0, "objective/kl": 36.50415039, "objective/entropy": -61.09580231, "objective/non_score_reward": -1.82520747, "objective/rlhf_reward": 12.39354229, "objective/scores": 14.25, "policy/approxkl_avg": 0.00081306, "policy/clipfrac_avg": 0.00510819, "loss/policy_avg": -0.01209749, "loss/value_avg": 0.700405, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16521072, "val/ratio": 0.98872697, "val/ratio_var": 6.137e-05, "val/num_eos_tokens": 182, "lr": 1.58e-06, "episode": 14928, "epoch": 0.75393939, "global_step/max_steps": "933/1238", "percentage": "75.36%", "elapsed_time": "11h 25m 38s", "remaining_time": "3h 44m 8s"} +{"eps": 0, "objective/kl": 34.97338867, "objective/entropy": -110.07270813, "objective/non_score_reward": -1.74866939, "objective/rlhf_reward": 14.54820538, "objective/scores": 16.25, "policy/approxkl_avg": 0.00091617, "policy/clipfrac_avg": 0.0067724, "loss/policy_avg": 0.03149443, "loss/value_avg": 0.33083206, "val/clipfrac_avg": 0.00159801, "policy/entropy_avg": 0.17972994, "val/ratio": 0.98782754, "val/ratio_var": 8.513e-05, "val/num_eos_tokens": 350, "lr": 1.57e-06, "episode": 14944, "epoch": 0.75474747, "global_step/max_steps": "934/1238", "percentage": "75.44%", "elapsed_time": "11h 26m 27s", "remaining_time": "3h 43m 25s"} +{"eps": 0, "objective/kl": 39.53004837, "objective/entropy": -25.03425026, "objective/non_score_reward": -1.97650254, "objective/rlhf_reward": 14.28131008, "objective/scores": 16.25, "policy/approxkl_avg": 0.00102007, "policy/clipfrac_avg": 0.004973, "loss/policy_avg": -0.03353989, "loss/value_avg": 0.41039872, "val/clipfrac_avg": 0.0242628, "policy/entropy_avg": 0.19387245, "val/ratio": 0.98752201, "val/ratio_var": 6.656e-05, "val/num_eos_tokens": 73, "lr": 1.56e-06, "episode": 14960, "epoch": 0.75555556, "global_step/max_steps": "935/1238", "percentage": "75.53%", "elapsed_time": "11h 26m 58s", "remaining_time": "3h 42m 37s"} +{"eps": 0, "objective/kl": 37.98503494, "objective/entropy": -60.77607346, "objective/non_score_reward": -1.8992517, "objective/rlhf_reward": 13.17106056, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00079483, "policy/clipfrac_avg": 0.00249668, "loss/policy_avg": -0.14741418, "loss/value_avg": 0.52727342, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15106249, "val/ratio": 0.99049866, "val/ratio_var": 3.401e-05, "val/num_eos_tokens": 96, "lr": 1.55e-06, "episode": 14976, "epoch": 0.75636364, "global_step/max_steps": "936/1238", "percentage": "75.61%", "elapsed_time": "11h 27m 33s", "remaining_time": "3h 41m 50s"} +{"eps": 0, "objective/kl": 41.95027161, "objective/entropy": -60.08255768, "objective/non_score_reward": -2.09751368, "objective/rlhf_reward": 12.6524868, "objective/scores": 14.75, "policy/approxkl_avg": 0.00092008, "policy/clipfrac_avg": 0.00597012, "loss/policy_avg": 0.13687547, "loss/value_avg": 0.62388521, "val/clipfrac_avg": 0.0124152, "policy/entropy_avg": 0.17743874, "val/ratio": 0.98798156, "val/ratio_var": 4.387e-05, "val/num_eos_tokens": 54, "lr": 1.54e-06, "episode": 14992, "epoch": 0.75717172, "global_step/max_steps": "937/1238", "percentage": "75.69%", "elapsed_time": "11h 28m 10s", "remaining_time": "3h 41m 4s"} +{"eps": 0, "objective/kl": 35.57097626, "objective/entropy": -112.66870117, "objective/non_score_reward": -1.77854896, "objective/rlhf_reward": 13.41676331, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00087959, "policy/clipfrac_avg": 0.00514485, "loss/policy_avg": -0.17327642, "loss/value_avg": 0.53824675, "val/clipfrac_avg": 0.04797268, "policy/entropy_avg": 0.16583538, "val/ratio": 0.99021786, "val/ratio_var": 5.214e-05, "val/num_eos_tokens": 136, "lr": 1.53e-06, "episode": 15008, "epoch": 0.7579798, "global_step/max_steps": "938/1238", "percentage": "75.77%", "elapsed_time": "11h 29m 3s", "remaining_time": "3h 40m 22s"} +{"eps": 0, "objective/kl": 35.88062286, "objective/entropy": -72.79998779, "objective/non_score_reward": -1.79403114, "objective/rlhf_reward": 14.46378231, "objective/scores": 16.25, "policy/approxkl_avg": 0.00085207, "policy/clipfrac_avg": 0.00303137, "loss/policy_avg": 0.59207451, "loss/value_avg": 1.14766943, "val/clipfrac_avg": 0.02116257, "policy/entropy_avg": 0.19948351, "val/ratio": 0.98822355, "val/ratio_var": 6.202e-05, "val/num_eos_tokens": 210, "lr": 1.52e-06, "episode": 15024, "epoch": 0.75878788, "global_step/max_steps": "939/1238", "percentage": "75.85%", "elapsed_time": "11h 29m 39s", "remaining_time": "3h 39m 36s"} +{"eps": 0, "objective/kl": 42.35734177, "objective/entropy": -91.86679077, "objective/non_score_reward": -2.11786699, "objective/rlhf_reward": 15.26494598, "objective/scores": 17.375, "policy/approxkl_avg": 0.00107419, "policy/clipfrac_avg": 0.00790455, "loss/policy_avg": -0.15639177, "loss/value_avg": 0.88131565, "val/clipfrac_avg": 0.015625, "policy/entropy_avg": 0.19293499, "val/ratio": 0.98698783, "val/ratio_var": 6.427e-05, "val/num_eos_tokens": 9, "lr": 1.51e-06, "episode": 15040, "epoch": 0.75959596, "global_step/max_steps": "940/1238", "percentage": "75.93%", "elapsed_time": "11h 30m 19s", "remaining_time": "3h 38m 50s"} +{"eps": 0, "objective/kl": 31.2521553, "objective/entropy": -62.3167038, "objective/non_score_reward": -1.562608, "objective/rlhf_reward": 14.15614223, "objective/scores": 15.75, "policy/approxkl_avg": 0.00071842, "policy/clipfrac_avg": 0.00353936, "loss/policy_avg": 0.21650478, "loss/value_avg": 0.34554335, "val/clipfrac_avg": 0.01320943, "policy/entropy_avg": 0.14359379, "val/ratio": 0.98960912, "val/ratio_var": 5.599e-05, "val/num_eos_tokens": 36, "lr": 1.5e-06, "episode": 15056, "epoch": 0.76040404, "global_step/max_steps": "941/1238", "percentage": "76.01%", "elapsed_time": "11h 30m 50s", "remaining_time": "3h 38m 2s"} +{"eps": 0, "objective/kl": 38.07510376, "objective/entropy": -109.56996918, "objective/non_score_reward": -1.90375519, "objective/rlhf_reward": 14.71343231, "objective/scores": 16.625, "policy/approxkl_avg": 0.00104122, "policy/clipfrac_avg": 0.00993063, "loss/policy_avg": -0.03007598, "loss/value_avg": 0.45760179, "val/clipfrac_avg": 0.01413989, "policy/entropy_avg": 0.18667316, "val/ratio": 0.98635983, "val/ratio_var": 0.00010635, "val/num_eos_tokens": 105, "lr": 1.49e-06, "episode": 15072, "epoch": 0.76121212, "global_step/max_steps": "942/1238", "percentage": "76.09%", "elapsed_time": "11h 31m 34s", "remaining_time": "3h 37m 18s"} +{"eps": 0, "objective/kl": 34.23678207, "objective/entropy": -121.37426758, "objective/non_score_reward": -1.7118392, "objective/rlhf_reward": 14.03816128, "objective/scores": 15.75, "policy/approxkl_avg": 0.00071554, "policy/clipfrac_avg": 0.00446352, "loss/policy_avg": 0.12722979, "loss/value_avg": 0.56963629, "val/clipfrac_avg": 0.0019725, "policy/entropy_avg": 0.13922501, "val/ratio": 0.99067104, "val/ratio_var": 6.089e-05, "val/num_eos_tokens": 493, "lr": 1.48e-06, "episode": 15088, "epoch": 0.7620202, "global_step/max_steps": "943/1238", "percentage": "76.17%", "elapsed_time": "11h 32m 18s", "remaining_time": "3h 36m 34s"} +{"eps": 0, "objective/kl": 42.60675812, "objective/entropy": -117.67542267, "objective/non_score_reward": -2.13033795, "objective/rlhf_reward": 13.85403633, "objective/scores": 16.0, "policy/approxkl_avg": 0.00073628, "policy/clipfrac_avg": 0.00928291, "loss/policy_avg": 0.2445948, "loss/value_avg": 0.99742723, "val/clipfrac_avg": 0.01938463, "policy/entropy_avg": 0.15163875, "val/ratio": 0.99071062, "val/ratio_var": 5.175e-05, "val/num_eos_tokens": 404, "lr": 1.47e-06, "episode": 15104, "epoch": 0.76282828, "global_step/max_steps": "944/1238", "percentage": "76.25%", "elapsed_time": "11h 33m 9s", "remaining_time": "3h 35m 52s"} +{"eps": 0, "objective/kl": 36.86358643, "objective/entropy": -72.4994278, "objective/non_score_reward": -1.84317923, "objective/rlhf_reward": 14.22713375, "objective/scores": 16.125, "policy/approxkl_avg": 0.00096557, "policy/clipfrac_avg": 0.0035611, "loss/policy_avg": 0.01902406, "loss/value_avg": 0.25053722, "val/clipfrac_avg": 0.00086178, "policy/entropy_avg": 0.19275665, "val/ratio": 0.98736012, "val/ratio_var": 7.995e-05, "val/num_eos_tokens": 33, "lr": 1.46e-06, "episode": 15120, "epoch": 0.76363636, "global_step/max_steps": "945/1238", "percentage": "76.33%", "elapsed_time": "11h 33m 39s", "remaining_time": "3h 35m 4s"} +{"eps": 0, "objective/kl": 30.41946602, "objective/entropy": -106.38670349, "objective/non_score_reward": -1.52097344, "objective/rlhf_reward": 15.33840179, "objective/scores": 16.875, "policy/approxkl_avg": 0.00090579, "policy/clipfrac_avg": 0.00906071, "loss/policy_avg": -0.24745053, "loss/value_avg": 0.40526709, "val/clipfrac_avg": 0.03885674, "policy/entropy_avg": 0.16122437, "val/ratio": 0.98913252, "val/ratio_var": 9.022e-05, "val/num_eos_tokens": 409, "lr": 1.46e-06, "episode": 15136, "epoch": 0.76444444, "global_step/max_steps": "946/1238", "percentage": "76.41%", "elapsed_time": "11h 34m 10s", "remaining_time": "3h 34m 16s"} +{"eps": 0, "objective/kl": 36.21892548, "objective/entropy": -104.1493988, "objective/non_score_reward": -1.81094623, "objective/rlhf_reward": 15.28280449, "objective/scores": 17.125, "policy/approxkl_avg": 0.00091934, "policy/clipfrac_avg": 0.01007547, "loss/policy_avg": -0.05101608, "loss/value_avg": 0.53373039, "val/clipfrac_avg": 0.00665455, "policy/entropy_avg": 0.16665363, "val/ratio": 0.98882133, "val/ratio_var": 8.222e-05, "val/num_eos_tokens": 424, "lr": 1.45e-06, "episode": 15152, "epoch": 0.76525253, "global_step/max_steps": "947/1238", "percentage": "76.49%", "elapsed_time": "11h 34m 50s", "remaining_time": "3h 33m 31s"} +{"eps": 0, "objective/kl": 37.63708496, "objective/entropy": -107.83410645, "objective/non_score_reward": -1.8818543, "objective/rlhf_reward": 11.46189594, "objective/scores": 13.375, "policy/approxkl_avg": 0.00088009, "policy/clipfrac_avg": 0.01894353, "loss/policy_avg": 0.5708428, "loss/value_avg": 0.91569042, "val/clipfrac_avg": 0.00067935, "policy/entropy_avg": 0.18418407, "val/ratio": 0.98806715, "val/ratio_var": 7.768e-05, "val/num_eos_tokens": 297, "lr": 1.44e-06, "episode": 15168, "epoch": 0.76606061, "global_step/max_steps": "948/1238", "percentage": "76.58%", "elapsed_time": "11h 35m 27s", "remaining_time": "3h 32m 44s"} +{"eps": 0, "objective/kl": 34.96649551, "objective/entropy": -11.56686878, "objective/non_score_reward": -1.74832475, "objective/rlhf_reward": 14.60323811, "objective/scores": 16.375, "policy/approxkl_avg": 0.00117357, "policy/clipfrac_avg": 0.00494196, "loss/policy_avg": -0.02806829, "loss/value_avg": 0.62192315, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2063942, "val/ratio": 0.98473334, "val/ratio_var": 3.737e-05, "val/num_eos_tokens": 32, "lr": 1.43e-06, "episode": 15184, "epoch": 0.76686869, "global_step/max_steps": "949/1238", "percentage": "76.66%", "elapsed_time": "11h 36m 3s", "remaining_time": "3h 31m 58s"} +{"eps": 0, "objective/kl": 31.67875481, "objective/entropy": -80.4765625, "objective/non_score_reward": -1.58393764, "objective/rlhf_reward": 13.85356236, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00068303, "policy/clipfrac_avg": 0.00417132, "loss/policy_avg": -0.15076536, "loss/value_avg": 0.27757215, "val/clipfrac_avg": 0.03938221, "policy/entropy_avg": 0.12936211, "val/ratio": 0.99113762, "val/ratio_var": 2.991e-05, "val/num_eos_tokens": 257, "lr": 1.42e-06, "episode": 15200, "epoch": 0.76767677, "global_step/max_steps": "950/1238", "percentage": "76.74%", "elapsed_time": "11h 36m 42s", "remaining_time": "3h 31m 12s"} +{"eps": 0, "objective/kl": 38.69972229, "objective/entropy": -84.18096161, "objective/non_score_reward": -1.93498623, "objective/rlhf_reward": 15.41657639, "objective/scores": 17.375, "policy/approxkl_avg": 0.00079864, "policy/clipfrac_avg": 0.00381712, "loss/policy_avg": -0.0464084, "loss/value_avg": 0.39227563, "val/clipfrac_avg": 0.00327317, "policy/entropy_avg": 0.17769951, "val/ratio": 0.98897815, "val/ratio_var": 5.715e-05, "val/num_eos_tokens": 326, "lr": 1.41e-06, "episode": 15216, "epoch": 0.76848485, "global_step/max_steps": "951/1238", "percentage": "76.82%", "elapsed_time": "11h 37m 24s", "remaining_time": "3h 30m 28s"} +{"eps": 0, "objective/kl": 39.91594315, "objective/entropy": -60.43904114, "objective/non_score_reward": -1.99579716, "objective/rlhf_reward": 14.2620163, "objective/scores": 16.25, "policy/approxkl_avg": 0.00097796, "policy/clipfrac_avg": 0.00458563, "loss/policy_avg": -0.12606917, "loss/value_avg": 0.31751049, "val/clipfrac_avg": 0.03066221, "policy/entropy_avg": 0.20082092, "val/ratio": 0.98622441, "val/ratio_var": 8.592e-05, "val/num_eos_tokens": 422, "lr": 1.4e-06, "episode": 15232, "epoch": 0.76929293, "global_step/max_steps": "952/1238", "percentage": "76.90%", "elapsed_time": "11h 37m 55s", "remaining_time": "3h 29m 40s"} +{"eps": 0, "objective/kl": 46.73610306, "objective/entropy": -67.14826965, "objective/non_score_reward": -2.33680511, "objective/rlhf_reward": 14.29600716, "objective/scores": 16.625, "policy/approxkl_avg": 0.0011254, "policy/clipfrac_avg": 0.0069552, "loss/policy_avg": 0.08768834, "loss/value_avg": 0.32770506, "val/clipfrac_avg": 8.927e-05, "policy/entropy_avg": 0.20795369, "val/ratio": 0.98547482, "val/ratio_var": 6.634e-05, "val/num_eos_tokens": 421, "lr": 1.39e-06, "episode": 15248, "epoch": 0.77010101, "global_step/max_steps": "953/1238", "percentage": "76.98%", "elapsed_time": "11h 38m 43s", "remaining_time": "3h 28m 57s"} +{"eps": 0, "objective/kl": 39.47554016, "objective/entropy": -52.92948914, "objective/non_score_reward": -1.97377706, "objective/rlhf_reward": 15.32309723, "objective/scores": 17.25, "policy/approxkl_avg": 0.00090011, "policy/clipfrac_avg": 0.0052912, "loss/policy_avg": 0.17172515, "loss/value_avg": 0.51825303, "val/clipfrac_avg": 0.00661506, "policy/entropy_avg": 0.1818161, "val/ratio": 0.98761833, "val/ratio_var": 5.797e-05, "val/num_eos_tokens": 490, "lr": 1.38e-06, "episode": 15264, "epoch": 0.77090909, "global_step/max_steps": "954/1238", "percentage": "77.06%", "elapsed_time": "11h 39m 15s", "remaining_time": "3h 28m 9s"} +{"eps": 0, "objective/kl": 41.42398071, "objective/entropy": -93.25178528, "objective/non_score_reward": -2.07119894, "objective/rlhf_reward": 13.15536308, "objective/scores": 15.25, "policy/approxkl_avg": 0.00094699, "policy/clipfrac_avg": 0.00416496, "loss/policy_avg": 0.34710482, "loss/value_avg": 0.71861547, "val/clipfrac_avg": 0.04515677, "policy/entropy_avg": 0.18822861, "val/ratio": 0.98775113, "val/ratio_var": 7.3e-05, "val/num_eos_tokens": 611, "lr": 1.37e-06, "episode": 15280, "epoch": 0.77171717, "global_step/max_steps": "955/1238", "percentage": "77.14%", "elapsed_time": "11h 40m 20s", "remaining_time": "3h 27m 32s"} +{"eps": 0, "objective/kl": 31.2022438, "objective/entropy": -112.50715637, "objective/non_score_reward": -1.56011212, "objective/rlhf_reward": 13.799263, "objective/scores": 15.375, "policy/approxkl_avg": 0.00079529, "policy/clipfrac_avg": 0.00852312, "loss/policy_avg": 0.24007092, "loss/value_avg": 0.45168853, "val/clipfrac_avg": 0.06015516, "policy/entropy_avg": 0.14573526, "val/ratio": 0.98959136, "val/ratio_var": 7.849e-05, "val/num_eos_tokens": 383, "lr": 1.36e-06, "episode": 15296, "epoch": 0.77252525, "global_step/max_steps": "956/1238", "percentage": "77.22%", "elapsed_time": "11h 40m 59s", "remaining_time": "3h 26m 46s"} +{"eps": 0, "objective/kl": 44.71451569, "objective/entropy": -82.63303375, "objective/non_score_reward": -2.23572588, "objective/rlhf_reward": 13.8189621, "objective/scores": 16.0, "policy/approxkl_avg": 0.00067997, "policy/clipfrac_avg": 0.00306352, "loss/policy_avg": -0.03721754, "loss/value_avg": 0.24181744, "val/clipfrac_avg": 9.246e-05, "policy/entropy_avg": 0.14811516, "val/ratio": 0.99055099, "val/ratio_var": 4.011e-05, "val/num_eos_tokens": 341, "lr": 1.35e-06, "episode": 15312, "epoch": 0.77333333, "global_step/max_steps": "957/1238", "percentage": "77.30%", "elapsed_time": "11h 41m 35s", "remaining_time": "3h 26m 0s"} +{"eps": 0, "objective/kl": 32.26376724, "objective/entropy": -75.73651886, "objective/non_score_reward": -1.61318827, "objective/rlhf_reward": 13.55087471, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00100095, "policy/clipfrac_avg": 0.00491299, "loss/policy_avg": -0.14485729, "loss/value_avg": 0.63617229, "val/clipfrac_avg": 0.00833672, "policy/entropy_avg": 0.19064331, "val/ratio": 0.98646599, "val/ratio_var": 9.686e-05, "val/num_eos_tokens": 329, "lr": 1.34e-06, "episode": 15328, "epoch": 0.77414141, "global_step/max_steps": "958/1238", "percentage": "77.38%", "elapsed_time": "11h 42m 8s", "remaining_time": "3h 25m 13s"} +{"eps": 0, "objective/kl": 39.02738953, "objective/entropy": -90.25643921, "objective/non_score_reward": -1.95136952, "objective/rlhf_reward": 14.95488071, "objective/scores": 16.875, "policy/approxkl_avg": 0.00090182, "policy/clipfrac_avg": 0.00477826, "loss/policy_avg": 0.11857907, "loss/value_avg": 0.88680494, "val/clipfrac_avg": 0.02706507, "policy/entropy_avg": 0.17965651, "val/ratio": 0.98826349, "val/ratio_var": 9.323e-05, "val/num_eos_tokens": 289, "lr": 1.33e-06, "episode": 15344, "epoch": 0.77494949, "global_step/max_steps": "959/1238", "percentage": "77.46%", "elapsed_time": "11h 42m 32s", "remaining_time": "3h 24m 23s"} +{"eps": 0, "objective/kl": 37.34764099, "objective/entropy": -101.29171753, "objective/non_score_reward": -1.86738193, "objective/rlhf_reward": 14.3201189, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008544, "policy/clipfrac_avg": 0.00470506, "loss/policy_avg": -0.22503328, "loss/value_avg": 0.48100311, "val/clipfrac_avg": 0.00054052, "policy/entropy_avg": 0.17150402, "val/ratio": 0.98826444, "val/ratio_var": 6.539e-05, "val/num_eos_tokens": 130, "lr": 1.33e-06, "episode": 15360, "epoch": 0.77575758, "global_step/max_steps": "960/1238", "percentage": "77.54%", "elapsed_time": "11h 43m 14s", "remaining_time": "3h 23m 38s"} +{"eps": 0, "objective/kl": 42.21212006, "objective/entropy": -97.00421143, "objective/non_score_reward": -2.11060619, "objective/rlhf_reward": 13.99876881, "objective/scores": 16.125, "policy/approxkl_avg": 0.00094652, "policy/clipfrac_avg": 0.00590877, "loss/policy_avg": -0.04760137, "loss/value_avg": 0.68238896, "val/clipfrac_avg": 0.0003125, "policy/entropy_avg": 0.19147015, "val/ratio": 0.98744112, "val/ratio_var": 5.347e-05, "val/num_eos_tokens": 224, "lr": 1.32e-06, "episode": 15376, "epoch": 0.77656566, "global_step/max_steps": "961/1238", "percentage": "77.63%", "elapsed_time": "11h 43m 56s", "remaining_time": "3h 22m 54s"} +{"eps": 0, "objective/kl": 39.2113533, "objective/entropy": -60.2877121, "objective/non_score_reward": -1.96056771, "objective/rlhf_reward": 15.35193253, "objective/scores": 17.25, "policy/approxkl_avg": 0.00096006, "policy/clipfrac_avg": 0.00359599, "loss/policy_avg": -0.26426494, "loss/value_avg": 0.47233304, "val/clipfrac_avg": 0.01204032, "policy/entropy_avg": 0.18239498, "val/ratio": 0.98753393, "val/ratio_var": 6.486e-05, "val/num_eos_tokens": 296, "lr": 1.31e-06, "episode": 15392, "epoch": 0.77737374, "global_step/max_steps": "962/1238", "percentage": "77.71%", "elapsed_time": "11h 44m 30s", "remaining_time": "3h 22m 7s"} +{"eps": 0, "objective/kl": 38.37220001, "objective/entropy": -146.63661194, "objective/non_score_reward": -1.9186101, "objective/rlhf_reward": 13.35482788, "objective/scores": 15.25, "policy/approxkl_avg": 0.00072013, "policy/clipfrac_avg": 0.00228993, "loss/policy_avg": 0.04519583, "loss/value_avg": 0.75955534, "val/clipfrac_avg": 8.292e-05, "policy/entropy_avg": 0.1562885, "val/ratio": 0.99069893, "val/ratio_var": 6.186e-05, "val/num_eos_tokens": 521, "lr": 1.3e-06, "episode": 15408, "epoch": 0.77818182, "global_step/max_steps": "963/1238", "percentage": "77.79%", "elapsed_time": "11h 45m 10s", "remaining_time": "3h 21m 22s"} +{"eps": 0, "objective/kl": 36.48185349, "objective/entropy": -66.0881958, "objective/non_score_reward": -1.82409275, "objective/rlhf_reward": 14.26184464, "objective/scores": 16.125, "policy/approxkl_avg": 0.00098322, "policy/clipfrac_avg": 0.00673567, "loss/policy_avg": -0.14623457, "loss/value_avg": 0.35101253, "val/clipfrac_avg": 0.00302419, "policy/entropy_avg": 0.19070911, "val/ratio": 0.98722857, "val/ratio_var": 7.909e-05, "val/num_eos_tokens": 421, "lr": 1.29e-06, "episode": 15424, "epoch": 0.7789899, "global_step/max_steps": "964/1238", "percentage": "77.87%", "elapsed_time": "11h 45m 37s", "remaining_time": "3h 20m 33s"} +{"eps": 0, "objective/kl": 36.51701355, "objective/entropy": -48.76942825, "objective/non_score_reward": -1.82585073, "objective/rlhf_reward": 13.86164856, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008362, "policy/clipfrac_avg": 0.00668676, "loss/policy_avg": -0.16694663, "loss/value_avg": 0.63735878, "val/clipfrac_avg": 0.00994318, "policy/entropy_avg": 0.17627907, "val/ratio": 0.98926079, "val/ratio_var": 5.406e-05, "val/num_eos_tokens": 11, "lr": 1.28e-06, "episode": 15440, "epoch": 0.77979798, "global_step/max_steps": "965/1238", "percentage": "77.95%", "elapsed_time": "11h 46m 14s", "remaining_time": "3h 19m 47s"} +{"eps": 0, "objective/kl": 42.56238937, "objective/entropy": -59.34945297, "objective/non_score_reward": -2.12811947, "objective/rlhf_reward": 13.91875553, "objective/scores": 16.0, "policy/approxkl_avg": 0.00087103, "policy/clipfrac_avg": 0.01020778, "loss/policy_avg": -0.00508792, "loss/value_avg": 0.27122128, "val/clipfrac_avg": 0.01080247, "policy/entropy_avg": 0.17399788, "val/ratio": 0.98846686, "val/ratio_var": 4.567e-05, "val/num_eos_tokens": 246, "lr": 1.27e-06, "episode": 15456, "epoch": 0.78060606, "global_step/max_steps": "966/1238", "percentage": "78.03%", "elapsed_time": "11h 47m 3s", "remaining_time": "3h 19m 5s"} +{"eps": 0, "objective/kl": 29.2735424, "objective/entropy": -136.5965271, "objective/non_score_reward": -1.46367717, "objective/rlhf_reward": 14.08319759, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00078264, "policy/clipfrac_avg": 0.00629924, "loss/policy_avg": -0.50366205, "loss/value_avg": 0.39513153, "val/clipfrac_avg": 0.03271849, "policy/entropy_avg": 0.15417519, "val/ratio": 0.9902066, "val/ratio_var": 8.734e-05, "val/num_eos_tokens": 249, "lr": 1.26e-06, "episode": 15472, "epoch": 0.78141414, "global_step/max_steps": "967/1238", "percentage": "78.11%", "elapsed_time": "11h 47m 36s", "remaining_time": "3h 18m 18s"} +{"eps": 0, "objective/kl": 38.6555481, "objective/entropy": -29.31902504, "objective/non_score_reward": -1.93277764, "objective/rlhf_reward": 13.86409664, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00089808, "policy/clipfrac_avg": 0.00365679, "loss/policy_avg": -0.04834941, "loss/value_avg": 0.49886179, "val/clipfrac_avg": 5.388e-05, "policy/entropy_avg": 0.18476677, "val/ratio": 0.98796588, "val/ratio_var": 2.396e-05, "val/num_eos_tokens": 10, "lr": 1.25e-06, "episode": 15488, "epoch": 0.78222222, "global_step/max_steps": "968/1238", "percentage": "78.19%", "elapsed_time": "11h 48m 17s", "remaining_time": "3h 17m 33s"} +{"eps": 0, "objective/kl": 41.82144547, "objective/entropy": -206.95504761, "objective/non_score_reward": -2.09107232, "objective/rlhf_reward": 13.07689667, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00085098, "policy/clipfrac_avg": 0.00477981, "loss/policy_avg": -0.22175038, "loss/value_avg": 0.41004986, "val/clipfrac_avg": 0.00021062, "policy/entropy_avg": 0.15673447, "val/ratio": 0.98876309, "val/ratio_var": 6.604e-05, "val/num_eos_tokens": 481, "lr": 1.25e-06, "episode": 15504, "epoch": 0.7830303, "global_step/max_steps": "969/1238", "percentage": "78.27%", "elapsed_time": "11h 50m 0s", "remaining_time": "3h 17m 6s"} +{"eps": 0, "objective/kl": 38.07307053, "objective/entropy": -40.90023804, "objective/non_score_reward": -1.9036535, "objective/rlhf_reward": 14.37759686, "objective/scores": 16.25, "policy/approxkl_avg": 0.00107271, "policy/clipfrac_avg": 0.00433798, "loss/policy_avg": -0.02120068, "loss/value_avg": 0.69019938, "val/clipfrac_avg": 0.02858383, "policy/entropy_avg": 0.19841003, "val/ratio": 0.98630285, "val/ratio_var": 8.23e-05, "val/num_eos_tokens": 535, "lr": 1.24e-06, "episode": 15520, "epoch": 0.78383838, "global_step/max_steps": "970/1238", "percentage": "78.35%", "elapsed_time": "11h 50m 40s", "remaining_time": "3h 16m 21s"} +{"eps": 0, "objective/kl": 39.35569382, "objective/entropy": -77.65346527, "objective/non_score_reward": -1.96778476, "objective/rlhf_reward": 13.90721512, "objective/scores": 15.875, "policy/approxkl_avg": 0.00087037, "policy/clipfrac_avg": 0.00438288, "loss/policy_avg": -0.09917282, "loss/value_avg": 0.39910382, "val/clipfrac_avg": 0.00012401, "policy/entropy_avg": 0.1721096, "val/ratio": 0.98807454, "val/ratio_var": 5.318e-05, "val/num_eos_tokens": 175, "lr": 1.23e-06, "episode": 15536, "epoch": 0.78464646, "global_step/max_steps": "971/1238", "percentage": "78.43%", "elapsed_time": "11h 51m 22s", "remaining_time": "3h 15m 36s"} +{"eps": 0, "objective/kl": 41.14687729, "objective/entropy": -16.92827606, "objective/non_score_reward": -2.05734396, "objective/rlhf_reward": 15.44265556, "objective/scores": 17.5, "policy/approxkl_avg": 0.00109003, "policy/clipfrac_avg": 0.00866632, "loss/policy_avg": 0.08149961, "loss/value_avg": 0.61755389, "val/clipfrac_avg": 0.0090258, "policy/entropy_avg": 0.2063446, "val/ratio": 0.98547161, "val/ratio_var": 2.748e-05, "val/num_eos_tokens": 174, "lr": 1.22e-06, "episode": 15552, "epoch": 0.78545455, "global_step/max_steps": "972/1238", "percentage": "78.51%", "elapsed_time": "11h 51m 59s", "remaining_time": "3h 14m 50s"} +{"eps": 0, "objective/kl": 45.04676819, "objective/entropy": -63.132267, "objective/non_score_reward": -2.25233841, "objective/rlhf_reward": 14.45078659, "objective/scores": 16.75, "policy/approxkl_avg": 0.00096167, "policy/clipfrac_avg": 0.00461027, "loss/policy_avg": 0.35931149, "loss/value_avg": 0.73723954, "val/clipfrac_avg": 0.01688945, "policy/entropy_avg": 0.18568325, "val/ratio": 0.9877485, "val/ratio_var": 6.444e-05, "val/num_eos_tokens": 419, "lr": 1.21e-06, "episode": 15568, "epoch": 0.78626263, "global_step/max_steps": "973/1238", "percentage": "78.59%", "elapsed_time": "11h 52m 56s", "remaining_time": "3h 14m 10s"} +{"eps": 0, "objective/kl": 41.90284348, "objective/entropy": -98.3809967, "objective/non_score_reward": -2.09514213, "objective/rlhf_reward": 12.18610764, "objective/scores": 14.25, "policy/approxkl_avg": 0.00093281, "policy/clipfrac_avg": 0.00490629, "loss/policy_avg": 0.14888805, "loss/value_avg": 0.75441074, "val/clipfrac_avg": 0.08914319, "policy/entropy_avg": 0.1748848, "val/ratio": 0.98880357, "val/ratio_var": 6.763e-05, "val/num_eos_tokens": 193, "lr": 1.2e-06, "episode": 15584, "epoch": 0.78707071, "global_step/max_steps": "974/1238", "percentage": "78.68%", "elapsed_time": "11h 53m 38s", "remaining_time": "3h 13m 25s"} +{"eps": 0, "objective/kl": 33.80537796, "objective/entropy": -65.5473175, "objective/non_score_reward": -1.69026899, "objective/rlhf_reward": 13.43473053, "objective/scores": 15.125, "policy/approxkl_avg": 0.00100511, "policy/clipfrac_avg": 0.00539667, "loss/policy_avg": -0.0541698, "loss/value_avg": 0.56266129, "val/clipfrac_avg": 0.02293883, "policy/entropy_avg": 0.19644976, "val/ratio": 0.98612064, "val/ratio_var": 7.482e-05, "val/num_eos_tokens": 539, "lr": 1.19e-06, "episode": 15600, "epoch": 0.78787879, "global_step/max_steps": "975/1238", "percentage": "78.76%", "elapsed_time": "11h 54m 8s", "remaining_time": "3h 12m 38s"} +{"eps": 0, "objective/kl": 36.37082291, "objective/entropy": -29.0012207, "objective/non_score_reward": -1.81854129, "objective/rlhf_reward": 12.14239693, "objective/scores": 13.9375, "policy/approxkl_avg": 0.00113971, "policy/clipfrac_avg": 0.00312591, "loss/policy_avg": 0.05607308, "loss/value_avg": 0.48446062, "val/clipfrac_avg": 0.00542428, "policy/entropy_avg": 0.22170258, "val/ratio": 0.98559475, "val/ratio_var": 6.332e-05, "val/num_eos_tokens": 387, "lr": 1.18e-06, "episode": 15616, "epoch": 0.78868687, "global_step/max_steps": "976/1238", "percentage": "78.84%", "elapsed_time": "11h 54m 44s", "remaining_time": "3h 11m 51s"} +{"eps": 0, "objective/kl": 37.47486115, "objective/entropy": -60.56960678, "objective/non_score_reward": -1.87374306, "objective/rlhf_reward": 11.04617882, "objective/scores": 12.9375, "policy/approxkl_avg": 0.0009816, "policy/clipfrac_avg": 0.00521398, "loss/policy_avg": -0.15400168, "loss/value_avg": 0.8165977, "val/clipfrac_avg": 0.02216701, "policy/entropy_avg": 0.21601486, "val/ratio": 0.9861747, "val/ratio_var": 7.865e-05, "val/num_eos_tokens": 93, "lr": 1.18e-06, "episode": 15632, "epoch": 0.78949495, "global_step/max_steps": "977/1238", "percentage": "78.92%", "elapsed_time": "11h 55m 13s", "remaining_time": "3h 11m 4s"} +{"eps": 0, "objective/kl": 37.22594833, "objective/entropy": -56.80758667, "objective/non_score_reward": -1.86129761, "objective/rlhf_reward": 14.38870239, "objective/scores": 16.25, "policy/approxkl_avg": 0.00102808, "policy/clipfrac_avg": 0.00375052, "loss/policy_avg": -0.17256549, "loss/value_avg": 0.48723835, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20328999, "val/ratio": 0.98718983, "val/ratio_var": 9.372e-05, "val/num_eos_tokens": 128, "lr": 1.17e-06, "episode": 15648, "epoch": 0.79030303, "global_step/max_steps": "978/1238", "percentage": "79.00%", "elapsed_time": "11h 55m 43s", "remaining_time": "3h 10m 16s"} +{"eps": 0, "objective/kl": 34.59331512, "objective/entropy": -78.30638123, "objective/non_score_reward": -1.72966576, "objective/rlhf_reward": 15.48908424, "objective/scores": 17.25, "policy/approxkl_avg": 0.00089044, "policy/clipfrac_avg": 0.00305291, "loss/policy_avg": -0.27645433, "loss/value_avg": 0.33185577, "val/clipfrac_avg": 0.0947064, "policy/entropy_avg": 0.17419052, "val/ratio": 0.98813945, "val/ratio_var": 6.589e-05, "val/num_eos_tokens": 367, "lr": 1.16e-06, "episode": 15664, "epoch": 0.79111111, "global_step/max_steps": "979/1238", "percentage": "79.08%", "elapsed_time": "11h 56m 13s", "remaining_time": "3h 9m 28s"} +{"eps": 0, "objective/kl": 40.9421196, "objective/entropy": -146.66398621, "objective/non_score_reward": -2.04710627, "objective/rlhf_reward": 12.40992546, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00086197, "policy/clipfrac_avg": 0.00762171, "loss/policy_avg": 0.04954625, "loss/value_avg": 0.59929246, "val/clipfrac_avg": 0.0969752, "policy/entropy_avg": 0.15411758, "val/ratio": 0.98896849, "val/ratio_var": 5.158e-05, "val/num_eos_tokens": 405, "lr": 1.15e-06, "episode": 15680, "epoch": 0.79191919, "global_step/max_steps": "980/1238", "percentage": "79.16%", "elapsed_time": "11h 57m 17s", "remaining_time": "3h 8m 50s"} +{"eps": 0, "objective/kl": 35.79730225, "objective/entropy": -111.55574799, "objective/non_score_reward": -1.78986514, "objective/rlhf_reward": 13.76482201, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00092242, "policy/clipfrac_avg": 0.00732938, "loss/policy_avg": 0.71014285, "loss/value_avg": 1.03456783, "val/clipfrac_avg": 0.0015625, "policy/entropy_avg": 0.17370558, "val/ratio": 0.98856634, "val/ratio_var": 7.396e-05, "val/num_eos_tokens": 230, "lr": 1.14e-06, "episode": 15696, "epoch": 0.79272727, "global_step/max_steps": "981/1238", "percentage": "79.24%", "elapsed_time": "11h 57m 54s", "remaining_time": "3h 8m 4s"} +{"eps": 0, "objective/kl": 41.06027985, "objective/entropy": -136.7578125, "objective/non_score_reward": -2.05301404, "objective/rlhf_reward": 14.56417274, "objective/scores": 16.625, "policy/approxkl_avg": 0.00085627, "policy/clipfrac_avg": 0.00573718, "loss/policy_avg": -0.19415256, "loss/value_avg": 0.41676646, "val/clipfrac_avg": 0.00664052, "policy/entropy_avg": 0.17065239, "val/ratio": 0.98932683, "val/ratio_var": 6.983e-05, "val/num_eos_tokens": 935, "lr": 1.13e-06, "episode": 15712, "epoch": 0.79353535, "global_step/max_steps": "982/1238", "percentage": "79.32%", "elapsed_time": "11h 58m 45s", "remaining_time": "3h 7m 22s"} +{"eps": 0, "objective/kl": 38.72102356, "objective/entropy": -109.03129578, "objective/non_score_reward": -1.93605113, "objective/rlhf_reward": 14.47019863, "objective/scores": 16.375, "policy/approxkl_avg": 0.00077122, "policy/clipfrac_avg": 0.00299889, "loss/policy_avg": -0.1428111, "loss/value_avg": 0.21746702, "val/clipfrac_avg": 0.02051012, "policy/entropy_avg": 0.16396713, "val/ratio": 0.9890219, "val/ratio_var": 4.009e-05, "val/num_eos_tokens": 25, "lr": 1.12e-06, "episode": 15728, "epoch": 0.79434343, "global_step/max_steps": "983/1238", "percentage": "79.40%", "elapsed_time": "11h 59m 28s", "remaining_time": "3h 6m 38s"} +{"eps": 0, "objective/kl": 38.89892578, "objective/entropy": -6.63568592, "objective/non_score_reward": -1.94494641, "objective/rlhf_reward": 14.71130371, "objective/scores": 16.625, "policy/approxkl_avg": 0.00095138, "policy/clipfrac_avg": 0.00315743, "loss/policy_avg": 0.01044519, "loss/value_avg": 0.37888825, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19734001, "val/ratio": 0.98759317, "val/ratio_var": 5.874e-05, "val/num_eos_tokens": 21, "lr": 1.12e-06, "episode": 15744, "epoch": 0.79515152, "global_step/max_steps": "984/1238", "percentage": "79.48%", "elapsed_time": "11h 59m 53s", "remaining_time": "3h 5m 49s"} +{"eps": 0, "objective/kl": 31.57590866, "objective/entropy": -149.35876465, "objective/non_score_reward": -1.57879543, "objective/rlhf_reward": 15.75714111, "objective/scores": 17.375, "policy/approxkl_avg": 0.00071625, "policy/clipfrac_avg": 0.00788946, "loss/policy_avg": -0.28500208, "loss/value_avg": 0.42132545, "val/clipfrac_avg": 0.02366625, "policy/entropy_avg": 0.14484692, "val/ratio": 0.99089503, "val/ratio_var": 5.517e-05, "val/num_eos_tokens": 155, "lr": 1.11e-06, "episode": 15760, "epoch": 0.7959596, "global_step/max_steps": "985/1238", "percentage": "79.56%", "elapsed_time": "12h 0m 29s", "remaining_time": "3h 5m 3s"} +{"eps": 0, "objective/kl": 44.14389801, "objective/entropy": -90.30593872, "objective/non_score_reward": -2.20719481, "objective/rlhf_reward": 12.29280567, "objective/scores": 14.5, "policy/approxkl_avg": 0.00089045, "policy/clipfrac_avg": 0.00572213, "loss/policy_avg": -0.01694701, "loss/value_avg": 0.39592227, "val/clipfrac_avg": 0.02059747, "policy/entropy_avg": 0.15873718, "val/ratio": 0.98871958, "val/ratio_var": 5.095e-05, "val/num_eos_tokens": 327, "lr": 1.1e-06, "episode": 15776, "epoch": 0.79676768, "global_step/max_steps": "986/1238", "percentage": "79.64%", "elapsed_time": "12h 1m 32s", "remaining_time": "3h 4m 24s"} +{"eps": 0, "objective/kl": 42.64774323, "objective/entropy": -140.49357605, "objective/non_score_reward": -2.13238716, "objective/rlhf_reward": 13.51605034, "objective/scores": 15.625, "policy/approxkl_avg": 0.00088214, "policy/clipfrac_avg": 0.00563179, "loss/policy_avg": 0.02288829, "loss/value_avg": 0.36602235, "val/clipfrac_avg": 0.01209004, "policy/entropy_avg": 0.17005873, "val/ratio": 0.98863995, "val/ratio_var": 5.682e-05, "val/num_eos_tokens": 546, "lr": 1.09e-06, "episode": 15792, "epoch": 0.79757576, "global_step/max_steps": "987/1238", "percentage": "79.73%", "elapsed_time": "12h 2m 15s", "remaining_time": "3h 3m 40s"} +{"eps": 0, "objective/kl": 36.98407364, "objective/entropy": -53.48846054, "objective/non_score_reward": -1.84920382, "objective/rlhf_reward": 14.04923344, "objective/scores": 15.875, "policy/approxkl_avg": 0.00097892, "policy/clipfrac_avg": 0.00588435, "loss/policy_avg": 0.19011469, "loss/value_avg": 0.51147193, "val/clipfrac_avg": 0.01335343, "policy/entropy_avg": 0.2069006, "val/ratio": 0.98720378, "val/ratio_var": 6.569e-05, "val/num_eos_tokens": 353, "lr": 1.08e-06, "episode": 15808, "epoch": 0.79838384, "global_step/max_steps": "988/1238", "percentage": "79.81%", "elapsed_time": "12h 2m 46s", "remaining_time": "3h 2m 53s"} +{"eps": 0, "objective/kl": 37.3478241, "objective/entropy": -56.68682861, "objective/non_score_reward": -1.86739135, "objective/rlhf_reward": 13.05057716, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00083888, "policy/clipfrac_avg": 0.00641349, "loss/policy_avg": 0.17533319, "loss/value_avg": 0.37737256, "val/clipfrac_avg": 0.05541594, "policy/entropy_avg": 0.16259098, "val/ratio": 0.98853952, "val/ratio_var": 8.253e-05, "val/num_eos_tokens": 32, "lr": 1.07e-06, "episode": 15824, "epoch": 0.79919192, "global_step/max_steps": "989/1238", "percentage": "79.89%", "elapsed_time": "12h 3m 21s", "remaining_time": "3h 2m 7s"} +{"eps": 0, "objective/kl": 34.28784943, "objective/entropy": -84.58714294, "objective/non_score_reward": -1.71439266, "objective/rlhf_reward": 13.20748234, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00096691, "policy/clipfrac_avg": 0.00355747, "loss/policy_avg": -0.01354401, "loss/value_avg": 0.54907268, "val/clipfrac_avg": 0.01583821, "policy/entropy_avg": 0.18613052, "val/ratio": 0.98773015, "val/ratio_var": 5.967e-05, "val/num_eos_tokens": 115, "lr": 1.07e-06, "episode": 15840, "epoch": 0.8, "global_step/max_steps": "990/1238", "percentage": "79.97%", "elapsed_time": "12h 4m 23s", "remaining_time": "3h 1m 27s"} +{"eps": 0, "objective/kl": 35.57737732, "objective/entropy": -134.7545929, "objective/non_score_reward": -1.77886868, "objective/rlhf_reward": 15.01019382, "objective/scores": 16.75, "policy/approxkl_avg": 0.00085205, "policy/clipfrac_avg": 0.00703627, "loss/policy_avg": 0.17707303, "loss/value_avg": 0.24937718, "val/clipfrac_avg": 0.00050403, "policy/entropy_avg": 0.14983177, "val/ratio": 0.98984069, "val/ratio_var": 4.887e-05, "val/num_eos_tokens": 450, "lr": 1.06e-06, "episode": 15856, "epoch": 0.80080808, "global_step/max_steps": "991/1238", "percentage": "80.05%", "elapsed_time": "12h 5m 7s", "remaining_time": "3h 0m 43s"} +{"eps": 0, "objective/kl": 37.30508804, "objective/entropy": -99.90908813, "objective/non_score_reward": -1.86525452, "objective/rlhf_reward": 12.7597456, "objective/scores": 14.625, "policy/approxkl_avg": 0.00087096, "policy/clipfrac_avg": 0.00413112, "loss/policy_avg": -0.12317791, "loss/value_avg": 0.27385566, "val/clipfrac_avg": 0.01071953, "policy/entropy_avg": 0.17008209, "val/ratio": 0.98945057, "val/ratio_var": 6.905e-05, "val/num_eos_tokens": 376, "lr": 1.05e-06, "episode": 15872, "epoch": 0.80161616, "global_step/max_steps": "992/1238", "percentage": "80.13%", "elapsed_time": "12h 5m 44s", "remaining_time": "2h 59m 58s"} +{"eps": 0, "objective/kl": 39.1995163, "objective/entropy": -89.06427765, "objective/non_score_reward": -1.95997596, "objective/rlhf_reward": 14.5322113, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009216, "policy/clipfrac_avg": 0.00672512, "loss/policy_avg": -0.11878392, "loss/value_avg": 0.60123473, "val/clipfrac_avg": 0.003125, "policy/entropy_avg": 0.17652702, "val/ratio": 0.98826921, "val/ratio_var": 3.506e-05, "val/num_eos_tokens": 74, "lr": 1.04e-06, "episode": 15888, "epoch": 0.80242424, "global_step/max_steps": "993/1238", "percentage": "80.21%", "elapsed_time": "12h 6m 33s", "remaining_time": "2h 59m 15s"} +{"eps": 0, "objective/kl": 37.943367, "objective/entropy": -48.50525665, "objective/non_score_reward": -1.89716828, "objective/rlhf_reward": 15.64189434, "objective/scores": 17.5, "policy/approxkl_avg": 0.00100367, "policy/clipfrac_avg": 0.00612472, "loss/policy_avg": 0.11932457, "loss/value_avg": 0.28720468, "val/clipfrac_avg": 0.01019022, "policy/entropy_avg": 0.1822567, "val/ratio": 0.9874962, "val/ratio_var": 5.374e-05, "val/num_eos_tokens": 8, "lr": 1.03e-06, "episode": 15904, "epoch": 0.80323232, "global_step/max_steps": "994/1238", "percentage": "80.29%", "elapsed_time": "12h 7m 7s", "remaining_time": "2h 58m 29s"} +{"eps": 0, "objective/kl": 41.18897629, "objective/entropy": -108.57736206, "objective/non_score_reward": -2.05944872, "objective/rlhf_reward": 14.1827383, "objective/scores": 16.25, "policy/approxkl_avg": 0.00101833, "policy/clipfrac_avg": 0.00341689, "loss/policy_avg": 0.14462614, "loss/value_avg": 0.32472521, "val/clipfrac_avg": 0.00548538, "policy/entropy_avg": 0.20872879, "val/ratio": 0.98654258, "val/ratio_var": 8.348e-05, "val/num_eos_tokens": 8, "lr": 1.03e-06, "episode": 15920, "epoch": 0.8040404, "global_step/max_steps": "995/1238", "percentage": "80.37%", "elapsed_time": "12h 8m 10s", "remaining_time": "2h 57m 50s"} +{"eps": 0, "objective/kl": 45.49590302, "objective/entropy": -43.08762741, "objective/non_score_reward": -2.27479506, "objective/rlhf_reward": 13.91270447, "objective/scores": 16.25, "policy/approxkl_avg": 0.00103734, "policy/clipfrac_avg": 0.00660848, "loss/policy_avg": 0.28177798, "loss/value_avg": 0.49794072, "val/clipfrac_avg": 0.03890984, "policy/entropy_avg": 0.21666098, "val/ratio": 0.98615086, "val/ratio_var": 8.553e-05, "val/num_eos_tokens": 11, "lr": 1.02e-06, "episode": 15936, "epoch": 0.80484848, "global_step/max_steps": "996/1238", "percentage": "80.45%", "elapsed_time": "12h 8m 46s", "remaining_time": "2h 57m 4s"} +{"eps": 0, "objective/kl": 35.89957428, "objective/entropy": -114.15419769, "objective/non_score_reward": -1.79497874, "objective/rlhf_reward": 14.8690834, "objective/scores": 16.625, "policy/approxkl_avg": 0.00073675, "policy/clipfrac_avg": 0.00375868, "loss/policy_avg": -0.22048, "loss/value_avg": 0.57450187, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15613556, "val/ratio": 0.99024796, "val/ratio_var": 5.668e-05, "val/num_eos_tokens": 560, "lr": 1.01e-06, "episode": 15952, "epoch": 0.80565657, "global_step/max_steps": "997/1238", "percentage": "80.53%", "elapsed_time": "12h 9m 23s", "remaining_time": "2h 56m 18s"} +{"eps": 0, "objective/kl": 39.43698883, "objective/entropy": -112.42098999, "objective/non_score_reward": -1.97184956, "objective/rlhf_reward": 14.48127556, "objective/scores": 16.5, "policy/approxkl_avg": 0.00090287, "policy/clipfrac_avg": 0.00592076, "loss/policy_avg": -0.04391951, "loss/value_avg": 0.65168691, "val/clipfrac_avg": 0.01274278, "policy/entropy_avg": 0.18039513, "val/ratio": 0.98780668, "val/ratio_var": 8.427e-05, "val/num_eos_tokens": 515, "lr": 1e-06, "episode": 15968, "epoch": 0.80646465, "global_step/max_steps": "998/1238", "percentage": "80.61%", "elapsed_time": "12h 10m 5s", "remaining_time": "2h 55m 34s"} +{"eps": 0, "objective/kl": 35.11191559, "objective/entropy": -115.74237061, "objective/non_score_reward": -1.75559592, "objective/rlhf_reward": 14.10377884, "objective/scores": 15.875, "policy/approxkl_avg": 0.00065866, "policy/clipfrac_avg": 0.00648142, "loss/policy_avg": -0.57262516, "loss/value_avg": 0.632213, "val/clipfrac_avg": 0.01994163, "policy/entropy_avg": 0.13949728, "val/ratio": 0.99069023, "val/ratio_var": 5.261e-05, "val/num_eos_tokens": 81, "lr": 9.9e-07, "episode": 15984, "epoch": 0.80727273, "global_step/max_steps": "999/1238", "percentage": "80.69%", "elapsed_time": "12h 10m 38s", "remaining_time": "2h 54m 47s"} +{"eps": 0, "objective/kl": 36.11966705, "objective/entropy": -35.66822052, "objective/non_score_reward": -1.8059833, "objective/rlhf_reward": 12.84245396, "objective/scores": 14.625, "policy/approxkl_avg": 0.00084978, "policy/clipfrac_avg": 0.00411999, "loss/policy_avg": -0.15711394, "loss/value_avg": 0.85969013, "val/clipfrac_avg": 0.02271465, "policy/entropy_avg": 0.16556644, "val/ratio": 0.98794091, "val/ratio_var": 6.342e-05, "val/num_eos_tokens": 82, "lr": 9.8e-07, "episode": 16000, "epoch": 0.80808081, "global_step/max_steps": "1000/1238", "percentage": "80.78%", "elapsed_time": "12h 11m 2s", "remaining_time": "2h 53m 59s"} +{"eps": 0, "objective/kl": 33.91107178, "objective/entropy": -112.11347961, "objective/non_score_reward": -1.69555354, "objective/rlhf_reward": 13.30444717, "objective/scores": 15.0, "policy/approxkl_avg": 0.00084471, "policy/clipfrac_avg": 0.00435919, "loss/policy_avg": -0.10610048, "loss/value_avg": 0.55933273, "val/clipfrac_avg": 0.0036526, "policy/entropy_avg": 0.18085873, "val/ratio": 0.98889935, "val/ratio_var": 0.00012091, "val/num_eos_tokens": 37, "lr": 9.8e-07, "episode": 16016, "epoch": 0.80888889, "global_step/max_steps": "1001/1238", "percentage": "80.86%", "elapsed_time": "12h 12m 8s", "remaining_time": "2h 53m 20s"} +{"eps": 0, "objective/kl": 32.61180878, "objective/entropy": -143.82159424, "objective/non_score_reward": -1.63059044, "objective/rlhf_reward": 13.32253456, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00075772, "policy/clipfrac_avg": 0.00546865, "loss/policy_avg": -0.4011277, "loss/value_avg": 0.82691336, "val/clipfrac_avg": 0.01630435, "policy/entropy_avg": 0.14389372, "val/ratio": 0.99035591, "val/ratio_var": 5e-05, "val/num_eos_tokens": 153, "lr": 9.7e-07, "episode": 16032, "epoch": 0.80969697, "global_step/max_steps": "1002/1238", "percentage": "80.94%", "elapsed_time": "12h 12m 43s", "remaining_time": "2h 52m 34s"} +{"eps": 0, "objective/kl": 43.84602356, "objective/entropy": -41.01973724, "objective/non_score_reward": -2.19230127, "objective/rlhf_reward": 14.87019825, "objective/scores": 17.0, "policy/approxkl_avg": 0.0009248, "policy/clipfrac_avg": 0.00408474, "loss/policy_avg": -0.18046468, "loss/value_avg": 0.44063193, "val/clipfrac_avg": 0.01253926, "policy/entropy_avg": 0.19304085, "val/ratio": 0.98778772, "val/ratio_var": 4.676e-05, "val/num_eos_tokens": 425, "lr": 9.6e-07, "episode": 16048, "epoch": 0.81050505, "global_step/max_steps": "1003/1238", "percentage": "81.02%", "elapsed_time": "12h 13m 9s", "remaining_time": "2h 51m 46s"} +{"eps": 0, "objective/kl": 43.82553864, "objective/entropy": -115.09342957, "objective/non_score_reward": -2.19127703, "objective/rlhf_reward": 14.941535, "objective/scores": 17.125, "policy/approxkl_avg": 0.00086233, "policy/clipfrac_avg": 0.00458201, "loss/policy_avg": -0.03701538, "loss/value_avg": 0.43129408, "val/clipfrac_avg": 0.02018961, "policy/entropy_avg": 0.17541945, "val/ratio": 0.98801827, "val/ratio_var": 5.917e-05, "val/num_eos_tokens": 259, "lr": 9.5e-07, "episode": 16064, "epoch": 0.81131313, "global_step/max_steps": "1004/1238", "percentage": "81.10%", "elapsed_time": "12h 13m 54s", "remaining_time": "2h 51m 2s"} +{"eps": 0, "objective/kl": 30.43748474, "objective/entropy": -62.09997177, "objective/non_score_reward": -1.52187431, "objective/rlhf_reward": 13.71250153, "objective/scores": 15.25, "policy/approxkl_avg": 0.00079711, "policy/clipfrac_avg": 0.00818359, "loss/policy_avg": 0.30280349, "loss/value_avg": 0.78989494, "val/clipfrac_avg": 0.00398936, "policy/entropy_avg": 0.14601326, "val/ratio": 0.9896875, "val/ratio_var": 4.33e-05, "val/num_eos_tokens": 215, "lr": 9.5e-07, "episode": 16080, "epoch": 0.81212121, "global_step/max_steps": "1005/1238", "percentage": "81.18%", "elapsed_time": "12h 14m 19s", "remaining_time": "2h 50m 14s"} +{"eps": 0, "objective/kl": 33.71896362, "objective/entropy": -46.67482758, "objective/non_score_reward": -1.68594801, "objective/rlhf_reward": 14.86874008, "objective/scores": 16.5, "policy/approxkl_avg": 0.000858, "policy/clipfrac_avg": 0.00321038, "loss/policy_avg": -0.23599471, "loss/value_avg": 1.14371014, "val/clipfrac_avg": 0.01090745, "policy/entropy_avg": 0.19154704, "val/ratio": 0.98714256, "val/ratio_var": 6.926e-05, "val/num_eos_tokens": 259, "lr": 9.4e-07, "episode": 16096, "epoch": 0.81292929, "global_step/max_steps": "1006/1238", "percentage": "81.26%", "elapsed_time": "12h 14m 44s", "remaining_time": "2h 49m 26s"} +{"eps": 0, "objective/kl": 35.53752899, "objective/entropy": -87.60648346, "objective/non_score_reward": -1.77687657, "objective/rlhf_reward": 15.64499855, "objective/scores": 17.375, "policy/approxkl_avg": 0.00071495, "policy/clipfrac_avg": 0.00227172, "loss/policy_avg": -0.02023146, "loss/value_avg": 1.29943597, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14923286, "val/ratio": 0.98877287, "val/ratio_var": 5.388e-05, "val/num_eos_tokens": 12, "lr": 9.3e-07, "episode": 16112, "epoch": 0.81373737, "global_step/max_steps": "1007/1238", "percentage": "81.34%", "elapsed_time": "12h 15m 37s", "remaining_time": "2h 48m 44s"} +{"eps": 0, "objective/kl": 36.12407303, "objective/entropy": -63.0642395, "objective/non_score_reward": -1.8062036, "objective/rlhf_reward": 15.14692116, "objective/scores": 17.0, "policy/approxkl_avg": 0.00089708, "policy/clipfrac_avg": 0.00471527, "loss/policy_avg": 0.02262819, "loss/value_avg": 1.10843551, "val/clipfrac_avg": 0.00885417, "policy/entropy_avg": 0.17540026, "val/ratio": 0.98806036, "val/ratio_var": 8.214e-05, "val/num_eos_tokens": 540, "lr": 9.2e-07, "episode": 16128, "epoch": 0.81454545, "global_step/max_steps": "1008/1238", "percentage": "81.42%", "elapsed_time": "12h 16m 7s", "remaining_time": "2h 47m 57s"} +{"eps": 0, "objective/kl": 38.30896759, "objective/entropy": -104.08320618, "objective/non_score_reward": -1.91544855, "objective/rlhf_reward": 13.20173931, "objective/scores": 15.125, "policy/approxkl_avg": 0.00082895, "policy/clipfrac_avg": 0.00365081, "loss/policy_avg": -0.06129603, "loss/value_avg": 0.55846, "val/clipfrac_avg": 0.0135351, "policy/entropy_avg": 0.16502059, "val/ratio": 0.98921359, "val/ratio_var": 6.506e-05, "val/num_eos_tokens": 360, "lr": 9.1e-07, "episode": 16144, "epoch": 0.81535354, "global_step/max_steps": "1009/1238", "percentage": "81.50%", "elapsed_time": "12h 16m 50s", "remaining_time": "2h 47m 13s"} +{"eps": 0, "objective/kl": 42.83964539, "objective/entropy": -71.5161438, "objective/non_score_reward": -2.14198208, "objective/rlhf_reward": 14.70176792, "objective/scores": 16.875, "policy/approxkl_avg": 0.00094445, "policy/clipfrac_avg": 0.00294417, "loss/policy_avg": -0.06057693, "loss/value_avg": 0.28895012, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18251228, "val/ratio": 0.98742414, "val/ratio_var": 3.965e-05, "val/num_eos_tokens": 613, "lr": 9.1e-07, "episode": 16160, "epoch": 0.81616162, "global_step/max_steps": "1010/1238", "percentage": "81.58%", "elapsed_time": "12h 17m 27s", "remaining_time": "2h 46m 28s"} +{"eps": 0, "objective/kl": 43.46244812, "objective/entropy": -63.48155212, "objective/non_score_reward": -2.17312241, "objective/rlhf_reward": 14.65500259, "objective/scores": 16.875, "policy/approxkl_avg": 0.00090739, "policy/clipfrac_avg": 0.00508339, "loss/policy_avg": -0.00314318, "loss/value_avg": 0.58741462, "val/clipfrac_avg": 0.00033507, "policy/entropy_avg": 0.17241001, "val/ratio": 0.98781729, "val/ratio_var": 5.252e-05, "val/num_eos_tokens": 564, "lr": 9e-07, "episode": 16176, "epoch": 0.8169697, "global_step/max_steps": "1011/1238", "percentage": "81.66%", "elapsed_time": "12h 18m 16s", "remaining_time": "2h 45m 45s"} +{"eps": 0, "objective/kl": 30.50366974, "objective/entropy": -24.23276138, "objective/non_score_reward": -1.52518344, "objective/rlhf_reward": 14.45919132, "objective/scores": 16.0, "policy/approxkl_avg": 0.00095718, "policy/clipfrac_avg": 0.00443454, "loss/policy_avg": 0.02021318, "loss/value_avg": 0.35578164, "val/clipfrac_avg": 0.01539849, "policy/entropy_avg": 0.18828392, "val/ratio": 0.98782682, "val/ratio_var": 6.846e-05, "val/num_eos_tokens": 104, "lr": 8.9e-07, "episode": 16192, "epoch": 0.81777778, "global_step/max_steps": "1012/1238", "percentage": "81.74%", "elapsed_time": "12h 18m 44s", "remaining_time": "2h 44m 58s"} +{"eps": 0, "objective/kl": 35.86807251, "objective/entropy": -72.25382996, "objective/non_score_reward": -1.79340374, "objective/rlhf_reward": 14.18315887, "objective/scores": 16.0, "policy/approxkl_avg": 0.00086642, "policy/clipfrac_avg": 0.00329332, "loss/policy_avg": -0.06331466, "loss/value_avg": 0.60162288, "val/clipfrac_avg": 0.00692653, "policy/entropy_avg": 0.1776638, "val/ratio": 0.98780596, "val/ratio_var": 6.279e-05, "val/num_eos_tokens": 294, "lr": 8.8e-07, "episode": 16208, "epoch": 0.81858586, "global_step/max_steps": "1013/1238", "percentage": "81.83%", "elapsed_time": "12h 19m 17s", "remaining_time": "2h 44m 12s"} +{"eps": 0, "objective/kl": 30.21366119, "objective/entropy": -88.55311584, "objective/non_score_reward": -1.51068318, "objective/rlhf_reward": 13.74712944, "objective/scores": 15.25, "policy/approxkl_avg": 0.00079725, "policy/clipfrac_avg": 0.00806876, "loss/policy_avg": 0.06992897, "loss/value_avg": 0.47664809, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14175224, "val/ratio": 0.99060249, "val/ratio_var": 5.377e-05, "val/num_eos_tokens": 51, "lr": 8.8e-07, "episode": 16224, "epoch": 0.81939394, "global_step/max_steps": "1014/1238", "percentage": "81.91%", "elapsed_time": "12h 19m 42s", "remaining_time": "2h 43m 24s"} +{"eps": 0, "objective/kl": 36.98034668, "objective/entropy": -92.63037109, "objective/non_score_reward": -1.84901762, "objective/rlhf_reward": 14.06504536, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00085285, "policy/clipfrac_avg": 0.00332333, "loss/policy_avg": 0.11788128, "loss/value_avg": 0.52844584, "val/clipfrac_avg": 0.01330586, "policy/entropy_avg": 0.17418957, "val/ratio": 0.98963517, "val/ratio_var": 4.991e-05, "val/num_eos_tokens": 387, "lr": 8.7e-07, "episode": 16240, "epoch": 0.82020202, "global_step/max_steps": "1015/1238", "percentage": "81.99%", "elapsed_time": "12h 20m 21s", "remaining_time": "2h 42m 39s"} +{"eps": 0, "objective/kl": 44.77581024, "objective/entropy": -62.09973907, "objective/non_score_reward": -2.23879075, "objective/rlhf_reward": 12.68308449, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00104638, "policy/clipfrac_avg": 0.00542851, "loss/policy_avg": -0.04949586, "loss/value_avg": 0.34623447, "val/clipfrac_avg": 0.03059276, "policy/entropy_avg": 0.21701241, "val/ratio": 0.98587084, "val/ratio_var": 6.539e-05, "val/num_eos_tokens": 270, "lr": 8.6e-07, "episode": 16256, "epoch": 0.8210101, "global_step/max_steps": "1016/1238", "percentage": "82.07%", "elapsed_time": "12h 20m 49s", "remaining_time": "2h 41m 52s"} +{"eps": 0, "objective/kl": 34.46260834, "objective/entropy": -65.84484863, "objective/non_score_reward": -1.72313058, "objective/rlhf_reward": 13.92530727, "objective/scores": 15.625, "policy/approxkl_avg": 0.00092558, "policy/clipfrac_avg": 0.00542821, "loss/policy_avg": 0.03657815, "loss/value_avg": 0.68759555, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.194911, "val/ratio": 0.98777646, "val/ratio_var": 6.734e-05, "val/num_eos_tokens": 15, "lr": 8.5e-07, "episode": 16272, "epoch": 0.82181818, "global_step/max_steps": "1017/1238", "percentage": "82.15%", "elapsed_time": "12h 21m 18s", "remaining_time": "2h 41m 5s"} +{"eps": 0, "objective/kl": 41.79958344, "objective/entropy": -21.66584778, "objective/non_score_reward": -2.08997917, "objective/rlhf_reward": 14.79283333, "objective/scores": 16.875, "policy/approxkl_avg": 0.0010365, "policy/clipfrac_avg": 0.0051118, "loss/policy_avg": -0.0975642, "loss/value_avg": 0.59978598, "val/clipfrac_avg": 0.00841346, "policy/entropy_avg": 0.20884323, "val/ratio": 0.9861449, "val/ratio_var": 5.512e-05, "val/num_eos_tokens": 123, "lr": 8.5e-07, "episode": 16288, "epoch": 0.82262626, "global_step/max_steps": "1018/1238", "percentage": "82.23%", "elapsed_time": "12h 21m 56s", "remaining_time": "2h 40m 20s"} +{"eps": 0, "objective/kl": 40.70600891, "objective/entropy": -106.49121094, "objective/non_score_reward": -2.03530025, "objective/rlhf_reward": 13.74594975, "objective/scores": 15.75, "policy/approxkl_avg": 0.00083165, "policy/clipfrac_avg": 0.00575118, "loss/policy_avg": -0.06873839, "loss/value_avg": 0.28038079, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14927673, "val/ratio": 0.98967302, "val/ratio_var": 4.523e-05, "val/num_eos_tokens": 205, "lr": 8.4e-07, "episode": 16304, "epoch": 0.82343434, "global_step/max_steps": "1019/1238", "percentage": "82.31%", "elapsed_time": "12h 22m 47s", "remaining_time": "2h 39m 38s"} +{"eps": 0, "objective/kl": 32.72511292, "objective/entropy": -29.91206551, "objective/non_score_reward": -1.63625574, "objective/rlhf_reward": 13.70749474, "objective/scores": 15.375, "policy/approxkl_avg": 0.00095812, "policy/clipfrac_avg": 0.00440688, "loss/policy_avg": 0.01863804, "loss/value_avg": 0.72353578, "val/clipfrac_avg": 0.00976562, "policy/entropy_avg": 0.20275879, "val/ratio": 0.98866796, "val/ratio_var": 7.176e-05, "val/num_eos_tokens": 35, "lr": 8.3e-07, "episode": 16320, "epoch": 0.82424242, "global_step/max_steps": "1020/1238", "percentage": "82.39%", "elapsed_time": "12h 23m 18s", "remaining_time": "2h 38m 51s"} +{"eps": 0, "objective/kl": 44.23792267, "objective/entropy": -17.82298851, "objective/non_score_reward": -2.21189618, "objective/rlhf_reward": 13.40529156, "objective/scores": 15.625, "policy/approxkl_avg": 0.00098654, "policy/clipfrac_avg": 0.0043609, "loss/policy_avg": 0.15834375, "loss/value_avg": 0.97195566, "val/clipfrac_avg": 0.00995964, "policy/entropy_avg": 0.19144058, "val/ratio": 0.9864338, "val/ratio_var": 2.899e-05, "val/num_eos_tokens": 44, "lr": 8.2e-07, "episode": 16336, "epoch": 0.82505051, "global_step/max_steps": "1021/1238", "percentage": "82.47%", "elapsed_time": "12h 23m 51s", "remaining_time": "2h 38m 5s"} +{"eps": 0, "objective/kl": 41.61705017, "objective/entropy": -87.69083405, "objective/non_score_reward": -2.08085251, "objective/rlhf_reward": 14.26289749, "objective/scores": 16.375, "policy/approxkl_avg": 0.00089914, "policy/clipfrac_avg": 0.00451232, "loss/policy_avg": 0.08962171, "loss/value_avg": 0.22370219, "val/clipfrac_avg": 0.04466812, "policy/entropy_avg": 0.19003296, "val/ratio": 0.98745018, "val/ratio_var": 6.217e-05, "val/num_eos_tokens": 17, "lr": 8.2e-07, "episode": 16352, "epoch": 0.82585859, "global_step/max_steps": "1022/1238", "percentage": "82.55%", "elapsed_time": "12h 24m 37s", "remaining_time": "2h 37m 22s"} +{"eps": 0, "objective/kl": 37.85612488, "objective/entropy": -103.04541779, "objective/non_score_reward": -1.89280629, "objective/rlhf_reward": 14.25563049, "objective/scores": 16.125, "policy/approxkl_avg": 0.00091218, "policy/clipfrac_avg": 0.00455268, "loss/policy_avg": -0.6490851, "loss/value_avg": 0.61456102, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17924374, "val/ratio": 0.98859298, "val/ratio_var": 8.515e-05, "val/num_eos_tokens": 624, "lr": 8.1e-07, "episode": 16368, "epoch": 0.82666667, "global_step/max_steps": "1023/1238", "percentage": "82.63%", "elapsed_time": "12h 25m 22s", "remaining_time": "2h 36m 39s"} +{"eps": 0, "objective/kl": 33.62747192, "objective/entropy": -46.77300644, "objective/non_score_reward": -1.68137372, "objective/rlhf_reward": 14.5217514, "objective/scores": 16.25, "policy/approxkl_avg": 0.00099587, "policy/clipfrac_avg": 0.00357719, "loss/policy_avg": 0.09050679, "loss/value_avg": 1.2950151, "val/clipfrac_avg": 0.00343916, "policy/entropy_avg": 0.17354488, "val/ratio": 0.98748684, "val/ratio_var": 7.542e-05, "val/num_eos_tokens": 5, "lr": 8e-07, "episode": 16384, "epoch": 0.82747475, "global_step/max_steps": "1024/1238", "percentage": "82.71%", "elapsed_time": "12h 26m 3s", "remaining_time": "2h 35m 54s"} +{"eps": 0, "objective/kl": 40.92263031, "objective/entropy": -118.60977173, "objective/non_score_reward": -2.04613137, "objective/rlhf_reward": 14.55543137, "objective/scores": 16.625, "policy/approxkl_avg": 0.00073777, "policy/clipfrac_avg": 0.00469288, "loss/policy_avg": 0.09637929, "loss/value_avg": 0.34717354, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15400505, "val/ratio": 0.98983729, "val/ratio_var": 4.05e-05, "val/num_eos_tokens": 445, "lr": 8e-07, "episode": 16400, "epoch": 0.82828283, "global_step/max_steps": "1025/1238", "percentage": "82.79%", "elapsed_time": "12h 26m 46s", "remaining_time": "2h 35m 11s"} +{"eps": 0, "objective/kl": 47.65478516, "objective/entropy": 5.97334623, "objective/non_score_reward": -2.38273954, "objective/rlhf_reward": 15.88288498, "objective/scores": 18.25, "policy/approxkl_avg": 0.00119578, "policy/clipfrac_avg": 0.00382707, "loss/policy_avg": 0.00393822, "loss/value_avg": 0.12186453, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.24349213, "val/ratio": 0.98368806, "val/ratio_var": 3.133e-05, "val/num_eos_tokens": 63, "lr": 7.9e-07, "episode": 16416, "epoch": 0.82909091, "global_step/max_steps": "1026/1238", "percentage": "82.88%", "elapsed_time": "12h 27m 25s", "remaining_time": "2h 34m 26s"} +{"eps": 0, "objective/kl": 43.73109818, "objective/entropy": -145.87228394, "objective/non_score_reward": -2.18655491, "objective/rlhf_reward": 12.70797634, "objective/scores": 14.875, "policy/approxkl_avg": 0.00098666, "policy/clipfrac_avg": 0.00542569, "loss/policy_avg": -0.0081724, "loss/value_avg": 0.40194565, "val/clipfrac_avg": 0.0016861, "policy/entropy_avg": 0.21741676, "val/ratio": 0.9879362, "val/ratio_var": 7.765e-05, "val/num_eos_tokens": 2002, "lr": 7.8e-07, "episode": 16432, "epoch": 0.82989899, "global_step/max_steps": "1027/1238", "percentage": "82.96%", "elapsed_time": "12h 29m 5s", "remaining_time": "2h 33m 54s"} +{"eps": 0, "objective/kl": 30.83833122, "objective/entropy": -39.62914276, "objective/non_score_reward": -1.54191661, "objective/rlhf_reward": 13.91902065, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00089824, "policy/clipfrac_avg": 0.00415442, "loss/policy_avg": -0.04023036, "loss/value_avg": 0.28793555, "val/clipfrac_avg": 0.00036337, "policy/entropy_avg": 0.15132141, "val/ratio": 0.99003947, "val/ratio_var": 4.598e-05, "val/num_eos_tokens": 42, "lr": 7.7e-07, "episode": 16448, "epoch": 0.83070707, "global_step/max_steps": "1028/1238", "percentage": "83.04%", "elapsed_time": "12h 29m 52s", "remaining_time": "2h 33m 11s"} +{"eps": 0, "objective/kl": 39.66940308, "objective/entropy": -91.81086731, "objective/non_score_reward": -1.9834702, "objective/rlhf_reward": 14.73528004, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009473, "policy/clipfrac_avg": 0.00336324, "loss/policy_avg": -0.17712682, "loss/value_avg": 0.41484764, "val/clipfrac_avg": 0.01510983, "policy/entropy_avg": 0.18948269, "val/ratio": 0.98733723, "val/ratio_var": 6.729e-05, "val/num_eos_tokens": 637, "lr": 7.7e-07, "episode": 16464, "epoch": 0.83151515, "global_step/max_steps": "1029/1238", "percentage": "83.12%", "elapsed_time": "12h 30m 33s", "remaining_time": "2h 32m 26s"} +{"eps": 0, "objective/kl": 40.63905334, "objective/entropy": -81.9229126, "objective/non_score_reward": -2.03195262, "objective/rlhf_reward": 14.96023464, "objective/scores": 17.0, "policy/approxkl_avg": 0.0009109, "policy/clipfrac_avg": 0.00393874, "loss/policy_avg": 0.03287977, "loss/value_avg": 0.36571223, "val/clipfrac_avg": 0.005359, "policy/entropy_avg": 0.17749858, "val/ratio": 0.98917335, "val/ratio_var": 5.904e-05, "val/num_eos_tokens": 56, "lr": 7.6e-07, "episode": 16480, "epoch": 0.83232323, "global_step/max_steps": "1030/1238", "percentage": "83.20%", "elapsed_time": "12h 31m 8s", "remaining_time": "2h 31m 41s"} +{"eps": 0, "objective/kl": 43.41844177, "objective/entropy": -79.71716309, "objective/non_score_reward": -2.17092204, "objective/rlhf_reward": 13.88376522, "objective/scores": 16.0, "policy/approxkl_avg": 0.00098977, "policy/clipfrac_avg": 0.00460906, "loss/policy_avg": 0.05487408, "loss/value_avg": 0.28462014, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18915176, "val/ratio": 0.98773724, "val/ratio_var": 6.537e-05, "val/num_eos_tokens": 413, "lr": 7.5e-07, "episode": 16496, "epoch": 0.83313131, "global_step/max_steps": "1031/1238", "percentage": "83.28%", "elapsed_time": "12h 31m 43s", "remaining_time": "2h 30m 55s"} +{"eps": 0, "objective/kl": 32.54696655, "objective/entropy": -75.63118744, "objective/non_score_reward": -1.62734842, "objective/rlhf_reward": 12.07577705, "objective/scores": 13.6875, "policy/approxkl_avg": 0.00095511, "policy/clipfrac_avg": 0.00577567, "loss/policy_avg": 0.42096102, "loss/value_avg": 0.70959961, "val/clipfrac_avg": 0.01792178, "policy/entropy_avg": 0.16732216, "val/ratio": 0.98902059, "val/ratio_var": 4.018e-05, "val/num_eos_tokens": 145, "lr": 7.5e-07, "episode": 16512, "epoch": 0.83393939, "global_step/max_steps": "1032/1238", "percentage": "83.36%", "elapsed_time": "12h 32m 34s", "remaining_time": "2h 30m 13s"} +{"eps": 0, "objective/kl": 38.59940338, "objective/entropy": -53.61748123, "objective/non_score_reward": -1.92997015, "objective/rlhf_reward": 12.64034176, "objective/scores": 14.5625, "policy/approxkl_avg": 0.00083903, "policy/clipfrac_avg": 0.00412296, "loss/policy_avg": 0.01750114, "loss/value_avg": 0.404511, "val/clipfrac_avg": 0.02054239, "policy/entropy_avg": 0.18261051, "val/ratio": 0.98851609, "val/ratio_var": 9.491e-05, "val/num_eos_tokens": 139, "lr": 7.4e-07, "episode": 16528, "epoch": 0.83474747, "global_step/max_steps": "1033/1238", "percentage": "83.44%", "elapsed_time": "12h 33m 6s", "remaining_time": "2h 29m 27s"} +{"eps": 0, "objective/kl": 36.21854401, "objective/entropy": -103.81524658, "objective/non_score_reward": -1.81092715, "objective/rlhf_reward": 14.07969856, "objective/scores": 15.875, "policy/approxkl_avg": 0.00076838, "policy/clipfrac_avg": 0.0030709, "loss/policy_avg": -0.03480758, "loss/value_avg": 1.4012475, "val/clipfrac_avg": 0.03183094, "policy/entropy_avg": 0.14527512, "val/ratio": 0.9899627, "val/ratio_var": 6.579e-05, "val/num_eos_tokens": 65, "lr": 7.3e-07, "episode": 16544, "epoch": 0.83555556, "global_step/max_steps": "1034/1238", "percentage": "83.52%", "elapsed_time": "12h 33m 36s", "remaining_time": "2h 28m 40s"} +{"eps": 0, "objective/kl": 38.71485901, "objective/entropy": -59.27926636, "objective/non_score_reward": -1.93574297, "objective/rlhf_reward": 14.07206917, "objective/scores": 16.0, "policy/approxkl_avg": 0.00075802, "policy/clipfrac_avg": 0.00303513, "loss/policy_avg": -0.28031889, "loss/value_avg": 1.12355232, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14683592, "val/ratio": 0.9899236, "val/ratio_var": 5.907e-05, "val/num_eos_tokens": 109, "lr": 7.2e-07, "episode": 16560, "epoch": 0.83636364, "global_step/max_steps": "1035/1238", "percentage": "83.60%", "elapsed_time": "12h 34m 9s", "remaining_time": "2h 27m 54s"} +{"eps": 0, "objective/kl": 38.85520554, "objective/entropy": -64.92012024, "objective/non_score_reward": -1.94276035, "objective/rlhf_reward": 14.40098953, "objective/scores": 16.375, "policy/approxkl_avg": 0.00095768, "policy/clipfrac_avg": 0.00488089, "loss/policy_avg": -0.14042839, "loss/value_avg": 1.11832631, "val/clipfrac_avg": 0.01285397, "policy/entropy_avg": 0.19240808, "val/ratio": 0.98732698, "val/ratio_var": 9.263e-05, "val/num_eos_tokens": 155, "lr": 7.2e-07, "episode": 16576, "epoch": 0.83717172, "global_step/max_steps": "1036/1238", "percentage": "83.68%", "elapsed_time": "12h 34m 40s", "remaining_time": "2h 27m 8s"} +{"eps": 0, "objective/kl": 36.47294617, "objective/entropy": -48.31360626, "objective/non_score_reward": -1.82364726, "objective/rlhf_reward": 14.13729, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00088762, "policy/clipfrac_avg": 0.00290118, "loss/policy_avg": -0.03311083, "loss/value_avg": 0.36772281, "val/clipfrac_avg": 0.0022611, "policy/entropy_avg": 0.20080185, "val/ratio": 0.98939776, "val/ratio_var": 5.76e-05, "val/num_eos_tokens": 20, "lr": 7.1e-07, "episode": 16592, "epoch": 0.8379798, "global_step/max_steps": "1037/1238", "percentage": "83.76%", "elapsed_time": "12h 35m 22s", "remaining_time": "2h 26m 24s"} +{"eps": 0, "objective/kl": 41.47463226, "objective/entropy": -66.3649292, "objective/non_score_reward": -2.07373142, "objective/rlhf_reward": 15.16845608, "objective/scores": 17.25, "policy/approxkl_avg": 0.00097564, "policy/clipfrac_avg": 0.00554347, "loss/policy_avg": -0.10436653, "loss/value_avg": 0.4317129, "val/clipfrac_avg": 0.01755008, "policy/entropy_avg": 0.1880188, "val/ratio": 0.98744977, "val/ratio_var": 5.146e-05, "val/num_eos_tokens": 3, "lr": 7e-07, "episode": 16608, "epoch": 0.83878788, "global_step/max_steps": "1038/1238", "percentage": "83.84%", "elapsed_time": "12h 35m 58s", "remaining_time": "2h 25m 39s"} +{"eps": 0, "objective/kl": 33.46930695, "objective/entropy": -82.42080688, "objective/non_score_reward": -1.67346525, "objective/rlhf_reward": 16.84997177, "objective/scores": 18.5, "policy/approxkl_avg": 0.00091661, "policy/clipfrac_avg": 0.0028332, "loss/policy_avg": -0.2336435, "loss/value_avg": 0.30871683, "val/clipfrac_avg": 0.02282338, "policy/entropy_avg": 0.18764019, "val/ratio": 0.98760146, "val/ratio_var": 7.869e-05, "val/num_eos_tokens": 150, "lr": 7e-07, "episode": 16624, "epoch": 0.83959596, "global_step/max_steps": "1039/1238", "percentage": "83.93%", "elapsed_time": "12h 36m 31s", "remaining_time": "2h 24m 53s"} +{"eps": 0, "objective/kl": 31.56005096, "objective/entropy": -98.47803497, "objective/non_score_reward": -1.57800245, "objective/rlhf_reward": 12.93762207, "objective/scores": 14.5, "policy/approxkl_avg": 0.00075816, "policy/clipfrac_avg": 0.00292523, "loss/policy_avg": -0.24348971, "loss/value_avg": 0.46729177, "val/clipfrac_avg": 0.01843094, "policy/entropy_avg": 0.14857721, "val/ratio": 0.99000621, "val/ratio_var": 7.637e-05, "val/num_eos_tokens": 64, "lr": 6.9e-07, "episode": 16640, "epoch": 0.84040404, "global_step/max_steps": "1040/1238", "percentage": "84.01%", "elapsed_time": "12h 37m 11s", "remaining_time": "2h 24m 9s"} +{"eps": 0, "objective/kl": 32.89226913, "objective/entropy": -67.75382996, "objective/non_score_reward": -1.6446135, "objective/rlhf_reward": 14.64444923, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009258, "policy/clipfrac_avg": 0.0047611, "loss/policy_avg": -0.30063081, "loss/value_avg": 0.9436385, "val/clipfrac_avg": 0.01880974, "policy/entropy_avg": 0.16883898, "val/ratio": 0.98938656, "val/ratio_var": 6.369e-05, "val/num_eos_tokens": 357, "lr": 6.8e-07, "episode": 16656, "epoch": 0.84121212, "global_step/max_steps": "1041/1238", "percentage": "84.09%", "elapsed_time": "12h 37m 45s", "remaining_time": "2h 23m 24s"} +{"eps": 0, "objective/kl": 35.5787468, "objective/entropy": -74.00183105, "objective/non_score_reward": -1.77893734, "objective/rlhf_reward": 13.29137421, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00088274, "policy/clipfrac_avg": 0.00451064, "loss/policy_avg": -0.06437137, "loss/value_avg": 0.42882502, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17483521, "val/ratio": 0.98894393, "val/ratio_var": 5.197e-05, "val/num_eos_tokens": 88, "lr": 6.8e-07, "episode": 16672, "epoch": 0.8420202, "global_step/max_steps": "1042/1238", "percentage": "84.17%", "elapsed_time": "12h 38m 21s", "remaining_time": "2h 22m 38s"} +{"eps": 0, "objective/kl": 36.04114914, "objective/entropy": -94.17185211, "objective/non_score_reward": -1.8020575, "objective/rlhf_reward": 14.45575523, "objective/scores": 16.25, "policy/approxkl_avg": 0.00072883, "policy/clipfrac_avg": 0.00380228, "loss/policy_avg": 0.08304592, "loss/value_avg": 0.62729943, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14253592, "val/ratio": 0.99007094, "val/ratio_var": 3.73e-05, "val/num_eos_tokens": 492, "lr": 6.7e-07, "episode": 16688, "epoch": 0.84282828, "global_step/max_steps": "1043/1238", "percentage": "84.25%", "elapsed_time": "12h 38m 53s", "remaining_time": "2h 21m 52s"} +{"eps": 0, "objective/kl": 37.80200958, "objective/entropy": -44.56480408, "objective/non_score_reward": -1.89010048, "objective/rlhf_reward": 13.20364952, "objective/scores": 15.125, "policy/approxkl_avg": 0.00120371, "policy/clipfrac_avg": 0.00529027, "loss/policy_avg": 0.28911531, "loss/value_avg": 0.6000362, "val/clipfrac_avg": 0.00351229, "policy/entropy_avg": 0.21977997, "val/ratio": 0.98471767, "val/ratio_var": 9.625e-05, "val/num_eos_tokens": 378, "lr": 6.6e-07, "episode": 16704, "epoch": 0.84363636, "global_step/max_steps": "1044/1238", "percentage": "84.33%", "elapsed_time": "12h 39m 27s", "remaining_time": "2h 21m 7s"} +{"eps": 0, "objective/kl": 39.69865799, "objective/entropy": -68.83721924, "objective/non_score_reward": -1.9849329, "objective/rlhf_reward": 13.4681921, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00087862, "policy/clipfrac_avg": 0.00498319, "loss/policy_avg": 0.07545819, "loss/value_avg": 0.4694804, "val/clipfrac_avg": 0.00048828, "policy/entropy_avg": 0.17647552, "val/ratio": 0.98801267, "val/ratio_var": 4.486e-05, "val/num_eos_tokens": 99, "lr": 6.6e-07, "episode": 16720, "epoch": 0.84444444, "global_step/max_steps": "1045/1238", "percentage": "84.41%", "elapsed_time": "12h 39m 58s", "remaining_time": "2h 20m 21s"} +{"eps": 0, "objective/kl": 39.10038376, "objective/entropy": -133.97775269, "objective/non_score_reward": -1.95501912, "objective/rlhf_reward": 13.5684185, "objective/scores": 15.5, "policy/approxkl_avg": 0.00072231, "policy/clipfrac_avg": 0.00235059, "loss/policy_avg": 0.18258721, "loss/value_avg": 0.33022794, "val/clipfrac_avg": 0.00100661, "policy/entropy_avg": 0.17428398, "val/ratio": 0.99031889, "val/ratio_var": 4.248e-05, "val/num_eos_tokens": 767, "lr": 6.5e-07, "episode": 16736, "epoch": 0.84525253, "global_step/max_steps": "1046/1238", "percentage": "84.49%", "elapsed_time": "12h 40m 40s", "remaining_time": "2h 19m 37s"} +{"eps": 0, "objective/kl": 40.72428894, "objective/entropy": -11.88162804, "objective/non_score_reward": -2.03621459, "objective/rlhf_reward": 15.11222267, "objective/scores": 17.125, "policy/approxkl_avg": 0.00105645, "policy/clipfrac_avg": 0.00793311, "loss/policy_avg": 0.00404117, "loss/value_avg": 0.25331676, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20357513, "val/ratio": 0.98680437, "val/ratio_var": 6.375e-05, "val/num_eos_tokens": 42, "lr": 6.4e-07, "episode": 16752, "epoch": 0.84606061, "global_step/max_steps": "1047/1238", "percentage": "84.57%", "elapsed_time": "12h 41m 11s", "remaining_time": "2h 18m 51s"} +{"eps": 0, "objective/kl": 34.73613739, "objective/entropy": -45.37768173, "objective/non_score_reward": -1.73680687, "objective/rlhf_reward": 14.19288063, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00098129, "policy/clipfrac_avg": 0.00497147, "loss/policy_avg": 0.07636248, "loss/value_avg": 0.43882942, "val/clipfrac_avg": 0.00061275, "policy/entropy_avg": 0.18614769, "val/ratio": 0.98753422, "val/ratio_var": 4.261e-05, "val/num_eos_tokens": 351, "lr": 6.4e-07, "episode": 16768, "epoch": 0.84686869, "global_step/max_steps": "1048/1238", "percentage": "84.65%", "elapsed_time": "12h 41m 42s", "remaining_time": "2h 18m 5s"} +{"eps": 0, "objective/kl": 42.18169022, "objective/entropy": -80.03951263, "objective/non_score_reward": -2.10908461, "objective/rlhf_reward": 13.81279087, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00094521, "policy/clipfrac_avg": 0.00481419, "loss/policy_avg": -0.1299208, "loss/value_avg": 0.93997085, "val/clipfrac_avg": 0.0003125, "policy/entropy_avg": 0.18466282, "val/ratio": 0.98873132, "val/ratio_var": 6.252e-05, "val/num_eos_tokens": 193, "lr": 6.3e-07, "episode": 16784, "epoch": 0.84767677, "global_step/max_steps": "1049/1238", "percentage": "84.73%", "elapsed_time": "12h 42m 18s", "remaining_time": "2h 17m 20s"} +{"eps": 0, "objective/kl": 37.25836945, "objective/entropy": -56.88700104, "objective/non_score_reward": -1.86291862, "objective/rlhf_reward": 14.30114365, "objective/scores": 16.125, "policy/approxkl_avg": 0.00080692, "policy/clipfrac_avg": 0.00219989, "loss/policy_avg": -0.12597322, "loss/value_avg": 0.59643769, "val/clipfrac_avg": 0.00014468, "policy/entropy_avg": 0.17313004, "val/ratio": 0.98892188, "val/ratio_var": 4.452e-05, "val/num_eos_tokens": 69, "lr": 6.2e-07, "episode": 16800, "epoch": 0.84848485, "global_step/max_steps": "1050/1238", "percentage": "84.81%", "elapsed_time": "12h 42m 59s", "remaining_time": "2h 16m 36s"} +{"eps": 0, "objective/kl": 36.81440735, "objective/entropy": -162.38363647, "objective/non_score_reward": -1.84072042, "objective/rlhf_reward": 13.54990482, "objective/scores": 15.375, "policy/approxkl_avg": 0.00068821, "policy/clipfrac_avg": 0.00293062, "loss/policy_avg": -0.28454351, "loss/value_avg": 0.49456328, "val/clipfrac_avg": 0.02376414, "policy/entropy_avg": 0.13589084, "val/ratio": 0.99121726, "val/ratio_var": 4.501e-05, "val/num_eos_tokens": 810, "lr": 6.2e-07, "episode": 16816, "epoch": 0.84929293, "global_step/max_steps": "1051/1238", "percentage": "84.89%", "elapsed_time": "12h 43m 43s", "remaining_time": "2h 15m 53s"} +{"eps": 0, "objective/kl": 35.95270538, "objective/entropy": -106.74056244, "objective/non_score_reward": -1.7976352, "objective/rlhf_reward": 16.46017838, "objective/scores": 18.25, "policy/approxkl_avg": 0.00090898, "policy/clipfrac_avg": 0.00528875, "loss/policy_avg": -0.0116379, "loss/value_avg": 0.55605364, "val/clipfrac_avg": 0.00260417, "policy/entropy_avg": 0.18283463, "val/ratio": 0.98826051, "val/ratio_var": 7.14e-05, "val/num_eos_tokens": 104, "lr": 6.1e-07, "episode": 16832, "epoch": 0.85010101, "global_step/max_steps": "1052/1238", "percentage": "84.98%", "elapsed_time": "12h 44m 19s", "remaining_time": "2h 15m 8s"} +{"eps": 0, "objective/kl": 40.82660294, "objective/entropy": -75.6595459, "objective/non_score_reward": -2.04133034, "objective/rlhf_reward": 13.85710716, "objective/scores": 15.875, "policy/approxkl_avg": 0.00091474, "policy/clipfrac_avg": 0.00465029, "loss/policy_avg": -0.09856722, "loss/value_avg": 0.43345025, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21721649, "val/ratio": 0.98730052, "val/ratio_var": 7.706e-05, "val/num_eos_tokens": 61, "lr": 6e-07, "episode": 16848, "epoch": 0.85090909, "global_step/max_steps": "1053/1238", "percentage": "85.06%", "elapsed_time": "12h 45m 4s", "remaining_time": "2h 14m 24s"} +{"eps": 0, "objective/kl": 38.76338959, "objective/entropy": -99.58453369, "objective/non_score_reward": -1.93816924, "objective/rlhf_reward": 15.05401802, "objective/scores": 17.0, "policy/approxkl_avg": 0.00084639, "policy/clipfrac_avg": 0.00381043, "loss/policy_avg": 0.05114016, "loss/value_avg": 0.46211517, "val/clipfrac_avg": 0.00191327, "policy/entropy_avg": 0.16744399, "val/ratio": 0.98934829, "val/ratio_var": 8.285e-05, "val/num_eos_tokens": 520, "lr": 6e-07, "episode": 16864, "epoch": 0.85171717, "global_step/max_steps": "1054/1238", "percentage": "85.14%", "elapsed_time": "12h 45m 43s", "remaining_time": "2h 13m 40s"} +{"eps": 0, "objective/kl": 37.57970047, "objective/entropy": -79.05471039, "objective/non_score_reward": -1.87898505, "objective/rlhf_reward": 14.56632805, "objective/scores": 16.5, "policy/approxkl_avg": 0.00096613, "policy/clipfrac_avg": 0.00524417, "loss/policy_avg": -0.03345665, "loss/value_avg": 0.4155297, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18662998, "val/ratio": 0.98819959, "val/ratio_var": 7.692e-05, "val/num_eos_tokens": 305, "lr": 5.9e-07, "episode": 16880, "epoch": 0.85252525, "global_step/max_steps": "1055/1238", "percentage": "85.22%", "elapsed_time": "12h 46m 19s", "remaining_time": "2h 12m 55s"} +{"eps": 0, "objective/kl": 38.44797516, "objective/entropy": -91.50765991, "objective/non_score_reward": -1.92239881, "objective/rlhf_reward": 12.52291393, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00078868, "policy/clipfrac_avg": 0.00426091, "loss/policy_avg": 0.18465675, "loss/value_avg": 0.58207226, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15495682, "val/ratio": 0.99095333, "val/ratio_var": 5.193e-05, "val/num_eos_tokens": 137, "lr": 5.9e-07, "episode": 16896, "epoch": 0.85333333, "global_step/max_steps": "1056/1238", "percentage": "85.30%", "elapsed_time": "12h 46m 48s", "remaining_time": "2h 12m 9s"} +{"eps": 0, "objective/kl": 41.44641876, "objective/entropy": -45.45872498, "objective/non_score_reward": -2.07232094, "objective/rlhf_reward": 13.96674156, "objective/scores": 16.0, "policy/approxkl_avg": 0.00094999, "policy/clipfrac_avg": 0.00787829, "loss/policy_avg": -0.01441033, "loss/value_avg": 0.45579588, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18343735, "val/ratio": 0.98878407, "val/ratio_var": 6.998e-05, "val/num_eos_tokens": 399, "lr": 5.8e-07, "episode": 16912, "epoch": 0.85414141, "global_step/max_steps": "1057/1238", "percentage": "85.38%", "elapsed_time": "12h 47m 17s", "remaining_time": "2h 11m 23s"} +{"eps": 0, "objective/kl": 38.32407379, "objective/entropy": -55.36639404, "objective/non_score_reward": -1.9162035, "objective/rlhf_reward": 12.029109, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0009155, "policy/clipfrac_avg": 0.00390601, "loss/policy_avg": -0.05935902, "loss/value_avg": 0.36739227, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17214489, "val/ratio": 0.98916751, "val/ratio_var": 5.269e-05, "val/num_eos_tokens": 62, "lr": 5.7e-07, "episode": 16928, "epoch": 0.85494949, "global_step/max_steps": "1058/1238", "percentage": "85.46%", "elapsed_time": "12h 47m 45s", "remaining_time": "2h 10m 37s"} +{"eps": 0, "objective/kl": 39.27944565, "objective/entropy": -77.39466858, "objective/non_score_reward": -1.96397245, "objective/rlhf_reward": 12.51259041, "objective/scores": 14.5, "policy/approxkl_avg": 0.00114888, "policy/clipfrac_avg": 0.00592923, "loss/policy_avg": 0.12509798, "loss/value_avg": 0.59017664, "val/clipfrac_avg": 0.0100913, "policy/entropy_avg": 0.22491837, "val/ratio": 0.98479104, "val/ratio_var": 8.845e-05, "val/num_eos_tokens": 274, "lr": 5.7e-07, "episode": 16944, "epoch": 0.85575758, "global_step/max_steps": "1059/1238", "percentage": "85.54%", "elapsed_time": "12h 48m 26s", "remaining_time": "2h 9m 53s"} +{"eps": 0, "objective/kl": 34.86209869, "objective/entropy": -68.7000351, "objective/non_score_reward": -1.74310493, "objective/rlhf_reward": 15.53814507, "objective/scores": 17.25, "policy/approxkl_avg": 0.00101325, "policy/clipfrac_avg": 0.00747871, "loss/policy_avg": 0.02743384, "loss/value_avg": 0.29364616, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19480419, "val/ratio": 0.98800987, "val/ratio_var": 6.574e-05, "val/num_eos_tokens": 6, "lr": 5.6e-07, "episode": 16960, "epoch": 0.85656566, "global_step/max_steps": "1060/1238", "percentage": "85.62%", "elapsed_time": "12h 49m 6s", "remaining_time": "2h 9m 9s"} +{"eps": 0, "objective/kl": 36.89395905, "objective/entropy": -101.62319183, "objective/non_score_reward": -1.84469819, "objective/rlhf_reward": 13.85061455, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00082188, "policy/clipfrac_avg": 0.00596947, "loss/policy_avg": 0.22168952, "loss/value_avg": 0.65441728, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15513206, "val/ratio": 0.98929024, "val/ratio_var": 6.282e-05, "val/num_eos_tokens": 465, "lr": 5.5e-07, "episode": 16976, "epoch": 0.85737374, "global_step/max_steps": "1061/1238", "percentage": "85.70%", "elapsed_time": "12h 49m 49s", "remaining_time": "2h 8m 25s"} +{"eps": 0, "objective/kl": 33.11014557, "objective/entropy": -112.72584534, "objective/non_score_reward": -1.65550709, "objective/rlhf_reward": 15.28980541, "objective/scores": 17.0, "policy/approxkl_avg": 0.00067529, "policy/clipfrac_avg": 0.00322104, "loss/policy_avg": -0.23859268, "loss/value_avg": 0.58530259, "val/clipfrac_avg": 4.234e-05, "policy/entropy_avg": 0.12448978, "val/ratio": 0.99157453, "val/ratio_var": 5.202e-05, "val/num_eos_tokens": 451, "lr": 5.5e-07, "episode": 16992, "epoch": 0.85818182, "global_step/max_steps": "1062/1238", "percentage": "85.78%", "elapsed_time": "12h 50m 24s", "remaining_time": "2h 7m 40s"} +{"eps": 0, "objective/kl": 38.63470078, "objective/entropy": -49.84119034, "objective/non_score_reward": -1.93173504, "objective/rlhf_reward": 16.25576401, "objective/scores": 18.25, "policy/approxkl_avg": 0.0011731, "policy/clipfrac_avg": 0.00805166, "loss/policy_avg": -0.14103599, "loss/value_avg": 0.22414374, "val/clipfrac_avg": 0.00904384, "policy/entropy_avg": 0.22179222, "val/ratio": 0.98546547, "val/ratio_var": 7.814e-05, "val/num_eos_tokens": 125, "lr": 5.4e-07, "episode": 17008, "epoch": 0.8589899, "global_step/max_steps": "1063/1238", "percentage": "85.86%", "elapsed_time": "12h 51m 2s", "remaining_time": "2h 6m 56s"} +{"eps": 0, "objective/kl": 37.95829773, "objective/entropy": -68.24198914, "objective/non_score_reward": -1.89791489, "objective/rlhf_reward": 15.38333511, "objective/scores": 17.25, "policy/approxkl_avg": 0.00086372, "policy/clipfrac_avg": 0.00695578, "loss/policy_avg": -0.06092814, "loss/value_avg": 0.45962244, "val/clipfrac_avg": 0.02772715, "policy/entropy_avg": 0.17585707, "val/ratio": 0.98864138, "val/ratio_var": 8.117e-05, "val/num_eos_tokens": 423, "lr": 5.4e-07, "episode": 17024, "epoch": 0.85979798, "global_step/max_steps": "1064/1238", "percentage": "85.95%", "elapsed_time": "12h 51m 37s", "remaining_time": "2h 6m 11s"} +{"eps": 0, "objective/kl": 41.04474258, "objective/entropy": -66.94715881, "objective/non_score_reward": -2.05223703, "objective/rlhf_reward": 13.01807594, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00096238, "policy/clipfrac_avg": 0.00379639, "loss/policy_avg": -0.02464094, "loss/value_avg": 0.73694032, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19322205, "val/ratio": 0.98774767, "val/ratio_var": 5.742e-05, "val/num_eos_tokens": 86, "lr": 5.3e-07, "episode": 17040, "epoch": 0.86060606, "global_step/max_steps": "1065/1238", "percentage": "86.03%", "elapsed_time": "12h 52m 12s", "remaining_time": "2h 5m 26s"} +{"eps": 0, "objective/kl": 40.31273651, "objective/entropy": -115.46347046, "objective/non_score_reward": -2.01563668, "objective/rlhf_reward": 13.36717606, "objective/scores": 15.375, "policy/approxkl_avg": 0.00084123, "policy/clipfrac_avg": 0.01834528, "loss/policy_avg": 0.11599922, "loss/value_avg": 1.5807066, "val/clipfrac_avg": 1.064e-05, "policy/entropy_avg": 0.17316818, "val/ratio": 0.98872864, "val/ratio_var": 4.432e-05, "val/num_eos_tokens": 71, "lr": 5.2e-07, "episode": 17056, "epoch": 0.86141414, "global_step/max_steps": "1066/1238", "percentage": "86.11%", "elapsed_time": "12h 53m 25s", "remaining_time": "2h 4m 47s"} +{"eps": 0, "objective/kl": 39.3602829, "objective/entropy": -67.14189148, "objective/non_score_reward": -1.968014, "objective/rlhf_reward": 15.41479874, "objective/scores": 17.375, "policy/approxkl_avg": 0.00089635, "policy/clipfrac_avg": 0.00728584, "loss/policy_avg": -0.11949857, "loss/value_avg": 0.53544647, "val/clipfrac_avg": 0.00721154, "policy/entropy_avg": 0.17730862, "val/ratio": 0.98818362, "val/ratio_var": 3.563e-05, "val/num_eos_tokens": 286, "lr": 5.2e-07, "episode": 17072, "epoch": 0.86222222, "global_step/max_steps": "1067/1238", "percentage": "86.19%", "elapsed_time": "12h 54m 1s", "remaining_time": "2h 4m 2s"} +{"eps": 0, "objective/kl": 37.19464493, "objective/entropy": -115.96305847, "objective/non_score_reward": -1.85973239, "objective/rlhf_reward": 12.88245583, "objective/scores": 14.75, "policy/approxkl_avg": 0.00063378, "policy/clipfrac_avg": 0.00421708, "loss/policy_avg": 0.1019949, "loss/value_avg": 0.35868531, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.13162804, "val/ratio": 0.99142712, "val/ratio_var": 5.448e-05, "val/num_eos_tokens": 156, "lr": 5.1e-07, "episode": 17088, "epoch": 0.8630303, "global_step/max_steps": "1068/1238", "percentage": "86.27%", "elapsed_time": "12h 54m 30s", "remaining_time": "2h 3m 16s"} +{"eps": 0, "objective/kl": 30.91879082, "objective/entropy": -27.45542526, "objective/non_score_reward": -1.54593956, "objective/rlhf_reward": 13.9696846, "objective/scores": 15.5, "policy/approxkl_avg": 0.00093203, "policy/clipfrac_avg": 0.00231644, "loss/policy_avg": 0.09657608, "loss/value_avg": 0.33049536, "val/clipfrac_avg": 0.00177103, "policy/entropy_avg": 0.20503426, "val/ratio": 0.98697788, "val/ratio_var": 7.318e-05, "val/num_eos_tokens": 117, "lr": 5.1e-07, "episode": 17104, "epoch": 0.86383838, "global_step/max_steps": "1069/1238", "percentage": "86.35%", "elapsed_time": "12h 55m 19s", "remaining_time": "2h 2m 34s"} +{"eps": 0, "objective/kl": 42.76515198, "objective/entropy": -41.6166954, "objective/non_score_reward": -2.1382575, "objective/rlhf_reward": 13.17424202, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00099013, "policy/clipfrac_avg": 0.00395975, "loss/policy_avg": -0.07673387, "loss/value_avg": 0.64296877, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19802952, "val/ratio": 0.98749125, "val/ratio_var": 4.667e-05, "val/num_eos_tokens": 199, "lr": 5e-07, "episode": 17120, "epoch": 0.86464646, "global_step/max_steps": "1070/1238", "percentage": "86.43%", "elapsed_time": "12h 55m 53s", "remaining_time": "2h 1m 49s"} +{"eps": 0, "objective/kl": 39.59283447, "objective/entropy": -53.61971283, "objective/non_score_reward": -1.9796418, "objective/rlhf_reward": 13.88754559, "objective/scores": 15.875, "policy/approxkl_avg": 0.00107037, "policy/clipfrac_avg": 0.00331488, "loss/policy_avg": -0.03734193, "loss/value_avg": 0.74408352, "val/clipfrac_avg": 0.00032552, "policy/entropy_avg": 0.19559097, "val/ratio": 0.98712701, "val/ratio_var": 6.861e-05, "val/num_eos_tokens": 134, "lr": 5e-07, "episode": 17136, "epoch": 0.86545455, "global_step/max_steps": "1071/1238", "percentage": "86.51%", "elapsed_time": "12h 56m 22s", "remaining_time": "2h 1m 3s"} +{"eps": 0, "objective/kl": 33.41657639, "objective/entropy": -32.19466782, "objective/non_score_reward": -1.67082882, "objective/rlhf_reward": 12.51667118, "objective/scores": 14.1875, "policy/approxkl_avg": 0.00105309, "policy/clipfrac_avg": 0.00505149, "loss/policy_avg": 0.05410761, "loss/value_avg": 0.62143338, "val/clipfrac_avg": 0.00698889, "policy/entropy_avg": 0.18621063, "val/ratio": 0.98618162, "val/ratio_var": 5.813e-05, "val/num_eos_tokens": 41, "lr": 4.9e-07, "episode": 17152, "epoch": 0.86626263, "global_step/max_steps": "1072/1238", "percentage": "86.59%", "elapsed_time": "12h 56m 40s", "remaining_time": "2h 0m 16s"} +{"eps": 0, "objective/kl": 44.27563858, "objective/entropy": -73.75119019, "objective/non_score_reward": -2.21378183, "objective/rlhf_reward": 13.87215614, "objective/scores": 16.125, "policy/approxkl_avg": 0.00097365, "policy/clipfrac_avg": 0.0021187, "loss/policy_avg": -0.13285211, "loss/value_avg": 0.3768208, "val/clipfrac_avg": 0.00336174, "policy/entropy_avg": 0.19892883, "val/ratio": 0.98648596, "val/ratio_var": 7.721e-05, "val/num_eos_tokens": 178, "lr": 4.8e-07, "episode": 17168, "epoch": 0.86707071, "global_step/max_steps": "1073/1238", "percentage": "86.67%", "elapsed_time": "12h 57m 9s", "remaining_time": "1h 59m 30s"} +{"eps": 0, "objective/kl": 29.79380035, "objective/entropy": -17.0313797, "objective/non_score_reward": -1.48969007, "objective/rlhf_reward": 15.44781017, "objective/scores": 17.0, "policy/approxkl_avg": 0.00086871, "policy/clipfrac_avg": 0.00158875, "loss/policy_avg": 0.03295011, "loss/value_avg": 0.56990969, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17066002, "val/ratio": 0.98956132, "val/ratio_var": 7.113e-05, "val/num_eos_tokens": 84, "lr": 4.8e-07, "episode": 17184, "epoch": 0.86787879, "global_step/max_steps": "1074/1238", "percentage": "86.75%", "elapsed_time": "12h 57m 39s", "remaining_time": "1h 58m 44s"} +{"eps": 0, "objective/kl": 39.95302582, "objective/entropy": -46.34983826, "objective/non_score_reward": -1.99765122, "objective/rlhf_reward": 14.6742239, "objective/scores": 16.625, "policy/approxkl_avg": 0.00085995, "policy/clipfrac_avg": 0.00693529, "loss/policy_avg": -0.11098457, "loss/value_avg": 0.53945893, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17142105, "val/ratio": 0.98907018, "val/ratio_var": 3.54e-05, "val/num_eos_tokens": 73, "lr": 4.7e-07, "episode": 17200, "epoch": 0.86868687, "global_step/max_steps": "1075/1238", "percentage": "86.83%", "elapsed_time": "12h 58m 18s", "remaining_time": "1h 58m 0s"} +{"eps": 0, "objective/kl": 43.934021, "objective/entropy": -62.08441544, "objective/non_score_reward": -2.19670105, "objective/rlhf_reward": 12.16267395, "objective/scores": 14.375, "policy/approxkl_avg": 0.00090521, "policy/clipfrac_avg": 0.00323779, "loss/policy_avg": 0.11330074, "loss/value_avg": 0.47666222, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18408585, "val/ratio": 0.98767948, "val/ratio_var": 4.134e-05, "val/num_eos_tokens": 588, "lr": 4.7e-07, "episode": 17216, "epoch": 0.86949495, "global_step/max_steps": "1076/1238", "percentage": "86.91%", "elapsed_time": "12h 59m 0s", "remaining_time": "1h 57m 17s"} +{"eps": 0, "objective/kl": 39.0636673, "objective/entropy": -73.24485779, "objective/non_score_reward": -1.95318353, "objective/rlhf_reward": 13.54681683, "objective/scores": 15.5, "policy/approxkl_avg": 0.00096865, "policy/clipfrac_avg": 0.00401127, "loss/policy_avg": -0.05885959, "loss/value_avg": 0.95601171, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18913555, "val/ratio": 0.98701757, "val/ratio_var": 8.096e-05, "val/num_eos_tokens": 200, "lr": 4.6e-07, "episode": 17232, "epoch": 0.87030303, "global_step/max_steps": "1077/1238", "percentage": "87.00%", "elapsed_time": "12h 59m 29s", "remaining_time": "1h 56m 31s"} +{"eps": 0, "objective/kl": 37.13741302, "objective/entropy": -116.47470093, "objective/non_score_reward": -1.85687089, "objective/rlhf_reward": 14.86187935, "objective/scores": 16.75, "policy/approxkl_avg": 0.00072711, "policy/clipfrac_avg": 0.00205629, "loss/policy_avg": 0.18471675, "loss/value_avg": 0.37227392, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16120768, "val/ratio": 0.99027705, "val/ratio_var": 6.446e-05, "val/num_eos_tokens": 178, "lr": 4.6e-07, "episode": 17248, "epoch": 0.87111111, "global_step/max_steps": "1078/1238", "percentage": "87.08%", "elapsed_time": "13h 0m 4s", "remaining_time": "1h 55m 46s"} +{"eps": 0, "objective/kl": 37.85641098, "objective/entropy": -93.82952881, "objective/non_score_reward": -1.8928206, "objective/rlhf_reward": 13.45874214, "objective/scores": 15.375, "policy/approxkl_avg": 0.00091645, "policy/clipfrac_avg": 0.00505855, "loss/policy_avg": -0.16059937, "loss/value_avg": 0.50746512, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15972978, "val/ratio": 0.98994964, "val/ratio_var": 8.054e-05, "val/num_eos_tokens": 44, "lr": 4.5e-07, "episode": 17264, "epoch": 0.87191919, "global_step/max_steps": "1079/1238", "percentage": "87.16%", "elapsed_time": "13h 0m 38s", "remaining_time": "1h 55m 2s"} +{"eps": 0, "objective/kl": 36.88376617, "objective/entropy": -93.29814911, "objective/non_score_reward": -1.84418845, "objective/rlhf_reward": 13.95268631, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00107461, "policy/clipfrac_avg": 0.00583021, "loss/policy_avg": 0.02416827, "loss/value_avg": 0.5276413, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17920113, "val/ratio": 0.98991513, "val/ratio_var": 4.746e-05, "val/num_eos_tokens": 519, "lr": 4.4e-07, "episode": 17280, "epoch": 0.87272727, "global_step/max_steps": "1080/1238", "percentage": "87.24%", "elapsed_time": "13h 1m 17s", "remaining_time": "1h 54m 17s"} +{"eps": 0, "objective/kl": 42.45173645, "objective/entropy": -163.22798157, "objective/non_score_reward": -2.12258673, "objective/rlhf_reward": 13.2367878, "objective/scores": 15.375, "policy/approxkl_avg": 0.00079639, "policy/clipfrac_avg": 0.00379314, "loss/policy_avg": -0.09750235, "loss/value_avg": 0.44001234, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1709981, "val/ratio": 0.98866749, "val/ratio_var": 8.438e-05, "val/num_eos_tokens": 294, "lr": 4.4e-07, "episode": 17296, "epoch": 0.87353535, "global_step/max_steps": "1081/1238", "percentage": "87.32%", "elapsed_time": "13h 2m 22s", "remaining_time": "1h 53m 37s"} +{"eps": 0, "objective/kl": 42.34510803, "objective/entropy": -63.06423569, "objective/non_score_reward": -2.11725521, "objective/rlhf_reward": 14.28899574, "objective/scores": 16.375, "policy/approxkl_avg": 0.00090703, "policy/clipfrac_avg": 0.00226762, "loss/policy_avg": -0.06599595, "loss/value_avg": 0.29759699, "val/clipfrac_avg": 0.0035636, "policy/entropy_avg": 0.18258286, "val/ratio": 0.98865396, "val/ratio_var": 6.365e-05, "val/num_eos_tokens": 576, "lr": 4.3e-07, "episode": 17312, "epoch": 0.87434343, "global_step/max_steps": "1082/1238", "percentage": "87.40%", "elapsed_time": "13h 2m 58s", "remaining_time": "1h 52m 53s"} +{"eps": 0, "objective/kl": 34.96723938, "objective/entropy": -124.63575745, "objective/non_score_reward": -1.74836206, "objective/rlhf_reward": 15.90007591, "objective/scores": 17.625, "policy/approxkl_avg": 0.000809, "policy/clipfrac_avg": 0.00315907, "loss/policy_avg": -0.12663837, "loss/value_avg": 1.61433959, "val/clipfrac_avg": 0.02003205, "policy/entropy_avg": 0.16525745, "val/ratio": 0.98906469, "val/ratio_var": 7.247e-05, "val/num_eos_tokens": 359, "lr": 4.3e-07, "episode": 17328, "epoch": 0.87515152, "global_step/max_steps": "1083/1238", "percentage": "87.48%", "elapsed_time": "13h 3m 33s", "remaining_time": "1h 52m 8s"} +{"eps": 0, "objective/kl": 44.47202301, "objective/entropy": -55.43729019, "objective/non_score_reward": -2.2236011, "objective/rlhf_reward": 13.72171211, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00094672, "policy/clipfrac_avg": 0.00637132, "loss/policy_avg": 0.10138821, "loss/value_avg": 0.35339892, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19496155, "val/ratio": 0.98646903, "val/ratio_var": 5.373e-05, "val/num_eos_tokens": 42, "lr": 4.2e-07, "episode": 17344, "epoch": 0.8759596, "global_step/max_steps": "1084/1238", "percentage": "87.56%", "elapsed_time": "13h 4m 5s", "remaining_time": "1h 51m 23s"} +{"eps": 0, "objective/kl": 35.82070541, "objective/entropy": -80.79370117, "objective/non_score_reward": -1.79103529, "objective/rlhf_reward": 13.5527153, "objective/scores": 15.375, "policy/approxkl_avg": 0.00079732, "policy/clipfrac_avg": 0.0033013, "loss/policy_avg": -0.16358021, "loss/value_avg": 0.76726878, "val/clipfrac_avg": 0.00211864, "policy/entropy_avg": 0.15000248, "val/ratio": 0.98962343, "val/ratio_var": 5.321e-05, "val/num_eos_tokens": 403, "lr": 4.2e-07, "episode": 17360, "epoch": 0.87676768, "global_step/max_steps": "1085/1238", "percentage": "87.64%", "elapsed_time": "13h 4m 49s", "remaining_time": "1h 50m 40s"} +{"eps": 0, "objective/kl": 33.16541672, "objective/entropy": -46.82775116, "objective/non_score_reward": -1.65827096, "objective/rlhf_reward": 14.40423012, "objective/scores": 16.0, "policy/approxkl_avg": 0.00082305, "policy/clipfrac_avg": 0.00301645, "loss/policy_avg": 0.20687677, "loss/value_avg": 1.36630046, "val/clipfrac_avg": 0.00264663, "policy/entropy_avg": 0.15550041, "val/ratio": 0.98926735, "val/ratio_var": 3.701e-05, "val/num_eos_tokens": 47, "lr": 4.1e-07, "episode": 17376, "epoch": 0.87757576, "global_step/max_steps": "1086/1238", "percentage": "87.72%", "elapsed_time": "13h 5m 18s", "remaining_time": "1h 49m 54s"} +{"eps": 0, "objective/kl": 45.35851288, "objective/entropy": -66.93091583, "objective/non_score_reward": -2.26792574, "objective/rlhf_reward": 14.12269974, "objective/scores": 16.375, "policy/approxkl_avg": 0.00089887, "policy/clipfrac_avg": 0.00436571, "loss/policy_avg": 0.18984848, "loss/value_avg": 0.31801367, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17530107, "val/ratio": 0.98793936, "val/ratio_var": 6.273e-05, "val/num_eos_tokens": 363, "lr": 4.1e-07, "episode": 17392, "epoch": 0.87838384, "global_step/max_steps": "1087/1238", "percentage": "87.80%", "elapsed_time": "13h 5m 50s", "remaining_time": "1h 49m 9s"} +{"eps": 0, "objective/kl": 44.8959465, "objective/entropy": -96.62719727, "objective/non_score_reward": -2.24479747, "objective/rlhf_reward": 14.02863979, "objective/scores": 16.25, "policy/approxkl_avg": 0.00095033, "policy/clipfrac_avg": 0.00471343, "loss/policy_avg": 0.14154401, "loss/value_avg": 0.2962133, "val/clipfrac_avg": 0.0040367, "policy/entropy_avg": 0.19094181, "val/ratio": 0.9875111, "val/ratio_var": 3.646e-05, "val/num_eos_tokens": 215, "lr": 4e-07, "episode": 17408, "epoch": 0.87919192, "global_step/max_steps": "1088/1238", "percentage": "87.88%", "elapsed_time": "13h 6m 34s", "remaining_time": "1h 48m 26s"} +{"eps": 0, "objective/kl": 44.09088898, "objective/entropy": -81.71890259, "objective/non_score_reward": -2.20454454, "objective/rlhf_reward": 14.31108093, "objective/scores": 16.5, "policy/approxkl_avg": 0.0008286, "policy/clipfrac_avg": 0.0029842, "loss/policy_avg": -0.27177691, "loss/value_avg": 0.65155411, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16189241, "val/ratio": 0.98929882, "val/ratio_var": 5.128e-05, "val/num_eos_tokens": 391, "lr": 4e-07, "episode": 17424, "epoch": 0.88, "global_step/max_steps": "1089/1238", "percentage": "87.96%", "elapsed_time": "13h 7m 7s", "remaining_time": "1h 47m 41s"} +{"eps": 0, "objective/kl": 45.33392334, "objective/entropy": -103.39527893, "objective/non_score_reward": -2.26669645, "objective/rlhf_reward": 15.03017902, "objective/scores": 17.25, "policy/approxkl_avg": 0.00083853, "policy/clipfrac_avg": 0.00225999, "loss/policy_avg": -0.09007428, "loss/value_avg": 0.17416966, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16774106, "val/ratio": 0.98831618, "val/ratio_var": 4.989e-05, "val/num_eos_tokens": 96, "lr": 3.9e-07, "episode": 17440, "epoch": 0.88080808, "global_step/max_steps": "1090/1238", "percentage": "88.05%", "elapsed_time": "13h 7m 56s", "remaining_time": "1h 46m 59s"} +{"eps": 0, "objective/kl": 38.23064423, "objective/entropy": -82.11065674, "objective/non_score_reward": -1.9115324, "objective/rlhf_reward": 13.2994051, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00087162, "policy/clipfrac_avg": 0.00727444, "loss/policy_avg": 0.29356396, "loss/value_avg": 0.87436247, "val/clipfrac_avg": 0.00089286, "policy/entropy_avg": 0.16114044, "val/ratio": 0.98829281, "val/ratio_var": 6.514e-05, "val/num_eos_tokens": 91, "lr": 3.9e-07, "episode": 17456, "epoch": 0.88161616, "global_step/max_steps": "1091/1238", "percentage": "88.13%", "elapsed_time": "13h 8m 34s", "remaining_time": "1h 46m 15s"} +{"eps": 0, "objective/kl": 37.99050522, "objective/entropy": -12.75318336, "objective/non_score_reward": -1.89952528, "objective/rlhf_reward": 12.65125561, "objective/scores": 14.5625, "policy/approxkl_avg": 0.00100711, "policy/clipfrac_avg": 0.00400542, "loss/policy_avg": 0.02994068, "loss/value_avg": 0.79247117, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19425201, "val/ratio": 0.98554111, "val/ratio_var": 4.669e-05, "val/num_eos_tokens": 14, "lr": 3.8e-07, "episode": 17472, "epoch": 0.88242424, "global_step/max_steps": "1092/1238", "percentage": "88.21%", "elapsed_time": "13h 9m 4s", "remaining_time": "1h 45m 29s"} +{"eps": 0, "objective/kl": 48.05560303, "objective/entropy": -36.1631279, "objective/non_score_reward": -2.40278053, "objective/rlhf_reward": 14.30034447, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009797, "policy/clipfrac_avg": 0.00579423, "loss/policy_avg": 0.13044786, "loss/value_avg": 0.37482464, "val/clipfrac_avg": 0.00260417, "policy/entropy_avg": 0.19660568, "val/ratio": 0.98657721, "val/ratio_var": 5.429e-05, "val/num_eos_tokens": 66, "lr": 3.8e-07, "episode": 17488, "epoch": 0.88323232, "global_step/max_steps": "1093/1238", "percentage": "88.29%", "elapsed_time": "13h 9m 47s", "remaining_time": "1h 44m 46s"} +{"eps": 0, "objective/kl": 45.28865051, "objective/entropy": -71.43814087, "objective/non_score_reward": -2.26443267, "objective/rlhf_reward": 13.72775459, "objective/scores": 16.0, "policy/approxkl_avg": 0.00095047, "policy/clipfrac_avg": 0.00271523, "loss/policy_avg": 0.07484121, "loss/value_avg": 0.26021606, "val/clipfrac_avg": 0.00067204, "policy/entropy_avg": 0.2066288, "val/ratio": 0.98663849, "val/ratio_var": 7.357e-05, "val/num_eos_tokens": 494, "lr": 3.7e-07, "episode": 17504, "epoch": 0.8840404, "global_step/max_steps": "1094/1238", "percentage": "88.37%", "elapsed_time": "13h 10m 21s", "remaining_time": "1h 44m 1s"} +{"eps": 0, "objective/kl": 39.92009354, "objective/entropy": -48.08239746, "objective/non_score_reward": -1.99600482, "objective/rlhf_reward": 12.98055744, "objective/scores": 15.0, "policy/approxkl_avg": 0.0010038, "policy/clipfrac_avg": 0.00439929, "loss/policy_avg": 0.1096379, "loss/value_avg": 0.7320236, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19763947, "val/ratio": 0.98702526, "val/ratio_var": 3.834e-05, "val/num_eos_tokens": 436, "lr": 3.7e-07, "episode": 17520, "epoch": 0.88484848, "global_step/max_steps": "1095/1238", "percentage": "88.45%", "elapsed_time": "13h 10m 52s", "remaining_time": "1h 43m 16s"} +{"eps": 0, "objective/kl": 41.74889374, "objective/entropy": -51.00461578, "objective/non_score_reward": -2.08744454, "objective/rlhf_reward": 14.54536819, "objective/scores": 16.625, "policy/approxkl_avg": 0.00108246, "policy/clipfrac_avg": 0.00963993, "loss/policy_avg": 0.05299535, "loss/value_avg": 0.27173871, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20147705, "val/ratio": 0.98759174, "val/ratio_var": 6.199e-05, "val/num_eos_tokens": 345, "lr": 3.6e-07, "episode": 17536, "epoch": 0.88565657, "global_step/max_steps": "1096/1238", "percentage": "88.53%", "elapsed_time": "13h 11m 29s", "remaining_time": "1h 42m 32s"} +{"eps": 0, "objective/kl": 41.82084656, "objective/entropy": -108.73769379, "objective/non_score_reward": -2.09104252, "objective/rlhf_reward": 14.15895844, "objective/scores": 16.25, "policy/approxkl_avg": 0.00075881, "policy/clipfrac_avg": 0.00366632, "loss/policy_avg": -0.0844707, "loss/value_avg": 0.3809917, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15442467, "val/ratio": 0.99024451, "val/ratio_var": 2.843e-05, "val/num_eos_tokens": 244, "lr": 3.6e-07, "episode": 17552, "epoch": 0.88646465, "global_step/max_steps": "1097/1238", "percentage": "88.61%", "elapsed_time": "13h 12m 7s", "remaining_time": "1h 41m 48s"} +{"eps": 0, "objective/kl": 33.65018082, "objective/entropy": -131.5652771, "objective/non_score_reward": -1.68250918, "objective/rlhf_reward": 13.74717903, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00090856, "policy/clipfrac_avg": 0.00498533, "loss/policy_avg": -0.4713923, "loss/value_avg": 1.42345679, "val/clipfrac_avg": 0.00024242, "policy/entropy_avg": 0.1699394, "val/ratio": 0.98880118, "val/ratio_var": 7.847e-05, "val/num_eos_tokens": 1220, "lr": 3.5e-07, "episode": 17568, "epoch": 0.88727273, "global_step/max_steps": "1098/1238", "percentage": "88.69%", "elapsed_time": "13h 13m 9s", "remaining_time": "1h 41m 7s"} +{"eps": 0, "objective/kl": 37.35829926, "objective/entropy": -42.34114075, "objective/non_score_reward": -1.86791515, "objective/rlhf_reward": 14.42895985, "objective/scores": 16.25, "policy/approxkl_avg": 0.00110664, "policy/clipfrac_avg": 0.0073922, "loss/policy_avg": 0.16145119, "loss/value_avg": 0.53603661, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21627808, "val/ratio": 0.98582065, "val/ratio_var": 6.102e-05, "val/num_eos_tokens": 172, "lr": 3.5e-07, "episode": 17584, "epoch": 0.88808081, "global_step/max_steps": "1099/1238", "percentage": "88.77%", "elapsed_time": "13h 13m 42s", "remaining_time": "1h 40m 23s"} +{"eps": 0, "objective/kl": 43.24127197, "objective/entropy": -32.03416824, "objective/non_score_reward": -2.16206384, "objective/rlhf_reward": 12.9394989, "objective/scores": 15.125, "policy/approxkl_avg": 0.00106291, "policy/clipfrac_avg": 0.00372437, "loss/policy_avg": 0.04367578, "loss/value_avg": 0.56874514, "val/clipfrac_avg": 0.00515095, "policy/entropy_avg": 0.20337868, "val/ratio": 0.9857685, "val/ratio_var": 4.003e-05, "val/num_eos_tokens": 106, "lr": 3.4e-07, "episode": 17600, "epoch": 0.88888889, "global_step/max_steps": "1100/1238", "percentage": "88.85%", "elapsed_time": "13h 14m 18s", "remaining_time": "1h 39m 38s"} +{"eps": 0, "objective/kl": 40.03705215, "objective/entropy": -65.2338028, "objective/non_score_reward": -2.00185251, "objective/rlhf_reward": 13.34970951, "objective/scores": 15.375, "policy/approxkl_avg": 0.00100419, "policy/clipfrac_avg": 0.00406351, "loss/policy_avg": -0.00186943, "loss/value_avg": 0.43309683, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19306087, "val/ratio": 0.98678124, "val/ratio_var": 5.167e-05, "val/num_eos_tokens": 14, "lr": 3.4e-07, "episode": 17616, "epoch": 0.88969697, "global_step/max_steps": "1101/1238", "percentage": "88.93%", "elapsed_time": "13h 15m 17s", "remaining_time": "1h 38m 57s"} +{"eps": 0, "objective/kl": 41.81010056, "objective/entropy": -81.65939331, "objective/non_score_reward": -2.09050512, "objective/rlhf_reward": 14.3938694, "objective/scores": 16.5, "policy/approxkl_avg": 0.00092539, "policy/clipfrac_avg": 0.00439174, "loss/policy_avg": 0.04897631, "loss/value_avg": 0.36726302, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18234062, "val/ratio": 0.98869598, "val/ratio_var": 5.684e-05, "val/num_eos_tokens": 128, "lr": 3.3e-07, "episode": 17632, "epoch": 0.89050505, "global_step/max_steps": "1102/1238", "percentage": "89.01%", "elapsed_time": "13h 16m 5s", "remaining_time": "1h 38m 14s"} +{"eps": 0, "objective/kl": 39.38318634, "objective/entropy": -97.10613251, "objective/non_score_reward": -1.96915936, "objective/rlhf_reward": 14.22615242, "objective/scores": 16.25, "policy/approxkl_avg": 0.00084514, "policy/clipfrac_avg": 0.00358358, "loss/policy_avg": 0.0288022, "loss/value_avg": 0.34563431, "val/clipfrac_avg": 0.00049761, "policy/entropy_avg": 0.182621, "val/ratio": 0.9880389, "val/ratio_var": 4.812e-05, "val/num_eos_tokens": 81, "lr": 3.3e-07, "episode": 17648, "epoch": 0.89131313, "global_step/max_steps": "1103/1238", "percentage": "89.10%", "elapsed_time": "13h 16m 54s", "remaining_time": "1h 37m 32s"} +{"eps": 0, "objective/kl": 39.81349182, "objective/entropy": -138.00392151, "objective/non_score_reward": -1.99067461, "objective/rlhf_reward": 13.14213753, "objective/scores": 15.125, "policy/approxkl_avg": 0.00093442, "policy/clipfrac_avg": 0.0041999, "loss/policy_avg": -0.1511742, "loss/value_avg": 1.13755345, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17568874, "val/ratio": 0.98694116, "val/ratio_var": 8.181e-05, "val/num_eos_tokens": 173, "lr": 3.2e-07, "episode": 17664, "epoch": 0.89212121, "global_step/max_steps": "1104/1238", "percentage": "89.18%", "elapsed_time": "13h 17m 50s", "remaining_time": "1h 36m 50s"} +{"eps": 0, "objective/kl": 38.82806396, "objective/entropy": -191.18865967, "objective/non_score_reward": -1.94140315, "objective/rlhf_reward": 13.94140816, "objective/scores": 15.875, "policy/approxkl_avg": 0.00059012, "policy/clipfrac_avg": 0.00535239, "loss/policy_avg": -0.40621763, "loss/value_avg": 0.46248072, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.12062049, "val/ratio": 0.99243033, "val/ratio_var": 4.216e-05, "val/num_eos_tokens": 236, "lr": 3.2e-07, "episode": 17680, "epoch": 0.89292929, "global_step/max_steps": "1105/1238", "percentage": "89.26%", "elapsed_time": "13h 18m 39s", "remaining_time": "1h 36m 7s"} +{"eps": 0, "objective/kl": 37.18630981, "objective/entropy": -87.46389008, "objective/non_score_reward": -1.85931551, "objective/rlhf_reward": 15.75787163, "objective/scores": 17.625, "policy/approxkl_avg": 0.00085626, "policy/clipfrac_avg": 0.00302501, "loss/policy_avg": -0.1159428, "loss/value_avg": 0.76151979, "val/clipfrac_avg": 8.562e-05, "policy/entropy_avg": 0.177001, "val/ratio": 0.98865283, "val/ratio_var": 7.095e-05, "val/num_eos_tokens": 557, "lr": 3.1e-07, "episode": 17696, "epoch": 0.89373737, "global_step/max_steps": "1106/1238", "percentage": "89.34%", "elapsed_time": "13h 19m 17s", "remaining_time": "1h 35m 23s"} +{"eps": 0, "objective/kl": 39.78605652, "objective/entropy": -110.63394928, "objective/non_score_reward": -1.98930311, "objective/rlhf_reward": 14.36225891, "objective/scores": 16.375, "policy/approxkl_avg": 0.00078054, "policy/clipfrac_avg": 0.00347289, "loss/policy_avg": -0.12702879, "loss/value_avg": 0.32193238, "val/clipfrac_avg": 0.00021701, "policy/entropy_avg": 0.14988577, "val/ratio": 0.98946387, "val/ratio_var": 6.243e-05, "val/num_eos_tokens": 344, "lr": 3.1e-07, "episode": 17712, "epoch": 0.89454545, "global_step/max_steps": "1107/1238", "percentage": "89.42%", "elapsed_time": "13h 19m 50s", "remaining_time": "1h 34m 39s"} +{"eps": 0, "objective/kl": 35.88131714, "objective/entropy": -74.73516846, "objective/non_score_reward": -1.79406571, "objective/rlhf_reward": 14.00280952, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00090876, "policy/clipfrac_avg": 0.00495595, "loss/policy_avg": -0.03015482, "loss/value_avg": 0.6115067, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19304085, "val/ratio": 0.98697001, "val/ratio_var": 6.041e-05, "val/num_eos_tokens": 300, "lr": 3e-07, "episode": 17728, "epoch": 0.89535354, "global_step/max_steps": "1108/1238", "percentage": "89.50%", "elapsed_time": "13h 20m 29s", "remaining_time": "1h 33m 55s"} +{"eps": 0, "objective/kl": 33.56185532, "objective/entropy": -105.74530029, "objective/non_score_reward": -1.67809296, "objective/rlhf_reward": 12.93909454, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009152, "policy/clipfrac_avg": 0.004283, "loss/policy_avg": -0.01582722, "loss/value_avg": 0.59545445, "val/clipfrac_avg": 0.00195312, "policy/entropy_avg": 0.18004417, "val/ratio": 0.98862648, "val/ratio_var": 6.606e-05, "val/num_eos_tokens": 107, "lr": 3e-07, "episode": 17744, "epoch": 0.89616162, "global_step/max_steps": "1109/1238", "percentage": "89.58%", "elapsed_time": "13h 21m 26s", "remaining_time": "1h 33m 13s"} +{"eps": 0, "objective/kl": 40.10255432, "objective/entropy": -105.83699799, "objective/non_score_reward": -2.00512791, "objective/rlhf_reward": 13.76830959, "objective/scores": 15.75, "policy/approxkl_avg": 0.0009933, "policy/clipfrac_avg": 0.00260761, "loss/policy_avg": -0.06135584, "loss/value_avg": 0.37346548, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19908524, "val/ratio": 0.98670614, "val/ratio_var": 8.14e-05, "val/num_eos_tokens": 189, "lr": 2.9e-07, "episode": 17760, "epoch": 0.8969697, "global_step/max_steps": "1110/1238", "percentage": "89.66%", "elapsed_time": "13h 22m 5s", "remaining_time": "1h 32m 29s"} +{"eps": 0, "objective/kl": 45.03960419, "objective/entropy": -69.66899109, "objective/non_score_reward": -2.2519803, "objective/rlhf_reward": 15.54489422, "objective/scores": 17.75, "policy/approxkl_avg": 0.00104727, "policy/clipfrac_avg": 0.0032264, "loss/policy_avg": 0.08733889, "loss/value_avg": 0.64302748, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20671892, "val/ratio": 0.98598707, "val/ratio_var": 6.778e-05, "val/num_eos_tokens": 714, "lr": 2.9e-07, "episode": 17776, "epoch": 0.89777778, "global_step/max_steps": "1111/1238", "percentage": "89.74%", "elapsed_time": "13h 22m 45s", "remaining_time": "1h 31m 45s"} +{"eps": 0, "objective/kl": 39.79743958, "objective/entropy": -89.71099091, "objective/non_score_reward": -1.9898721, "objective/rlhf_reward": 16.38512802, "objective/scores": 18.375, "policy/approxkl_avg": 0.00102079, "policy/clipfrac_avg": 0.00437536, "loss/policy_avg": -0.13560356, "loss/value_avg": 0.60800236, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19225121, "val/ratio": 0.98722398, "val/ratio_var": 7.31e-05, "val/num_eos_tokens": 296, "lr": 2.9e-07, "episode": 17792, "epoch": 0.89858586, "global_step/max_steps": "1112/1238", "percentage": "89.82%", "elapsed_time": "13h 23m 18s", "remaining_time": "1h 31m 1s"} +{"eps": 0, "objective/kl": 48.40119934, "objective/entropy": -31.37752533, "objective/non_score_reward": -2.42005992, "objective/rlhf_reward": 15.62681484, "objective/scores": 18.0, "policy/approxkl_avg": 0.00107731, "policy/clipfrac_avg": 0.00744739, "loss/policy_avg": -0.33778393, "loss/value_avg": 0.49397686, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21597433, "val/ratio": 0.98518246, "val/ratio_var": 7.279e-05, "val/num_eos_tokens": 28, "lr": 2.8e-07, "episode": 17808, "epoch": 0.89939394, "global_step/max_steps": "1113/1238", "percentage": "89.90%", "elapsed_time": "13h 23m 57s", "remaining_time": "1h 30m 17s"} +{"eps": 0, "objective/kl": 37.62234116, "objective/entropy": -30.16031265, "objective/non_score_reward": -1.88111711, "objective/rlhf_reward": 12.79857063, "objective/scores": 14.6875, "policy/approxkl_avg": 0.00105482, "policy/clipfrac_avg": 0.00434351, "loss/policy_avg": -0.10283504, "loss/value_avg": 0.59921706, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19416046, "val/ratio": 0.98757017, "val/ratio_var": 4.415e-05, "val/num_eos_tokens": 115, "lr": 2.8e-07, "episode": 17824, "epoch": 0.90020202, "global_step/max_steps": "1114/1238", "percentage": "89.98%", "elapsed_time": "13h 24m 21s", "remaining_time": "1h 29m 32s"} +{"eps": 0, "objective/kl": 44.19203568, "objective/entropy": -98.54946899, "objective/non_score_reward": -2.20960188, "objective/rlhf_reward": 12.51696014, "objective/scores": 14.75, "policy/approxkl_avg": 0.00085504, "policy/clipfrac_avg": 0.00352586, "loss/policy_avg": 0.15831202, "loss/value_avg": 0.50158322, "val/clipfrac_avg": 9.246e-05, "policy/entropy_avg": 0.17531872, "val/ratio": 0.98839653, "val/ratio_var": 6.708e-05, "val/num_eos_tokens": 277, "lr": 2.7e-07, "episode": 17840, "epoch": 0.9010101, "global_step/max_steps": "1115/1238", "percentage": "90.06%", "elapsed_time": "13h 24m 52s", "remaining_time": "1h 28m 47s"} +{"eps": 0, "objective/kl": 40.80606842, "objective/entropy": -89.1390686, "objective/non_score_reward": -2.04030323, "objective/rlhf_reward": 14.84250927, "objective/scores": 16.875, "policy/approxkl_avg": 0.00097494, "policy/clipfrac_avg": 0.00458724, "loss/policy_avg": -0.08836727, "loss/value_avg": 0.38530809, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18624973, "val/ratio": 0.98782289, "val/ratio_var": 6.972e-05, "val/num_eos_tokens": 42, "lr": 2.7e-07, "episode": 17856, "epoch": 0.90181818, "global_step/max_steps": "1116/1238", "percentage": "90.15%", "elapsed_time": "13h 25m 26s", "remaining_time": "1h 28m 3s"} +{"eps": 0, "objective/kl": 39.73638916, "objective/entropy": -94.74272156, "objective/non_score_reward": -1.98681939, "objective/rlhf_reward": 14.48974323, "objective/scores": 16.5, "policy/approxkl_avg": 0.00101438, "policy/clipfrac_avg": 0.00911447, "loss/policy_avg": 0.35861504, "loss/value_avg": 0.47818783, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18755674, "val/ratio": 0.9870187, "val/ratio_var": 6.581e-05, "val/num_eos_tokens": 316, "lr": 2.6e-07, "episode": 17872, "epoch": 0.90262626, "global_step/max_steps": "1117/1238", "percentage": "90.23%", "elapsed_time": "13h 26m 4s", "remaining_time": "1h 27m 19s"} +{"eps": 0, "objective/kl": 38.03620148, "objective/entropy": -60.80832291, "objective/non_score_reward": -1.90181029, "objective/rlhf_reward": 14.41850185, "objective/scores": 16.375, "policy/approxkl_avg": 0.00096647, "policy/clipfrac_avg": 0.00549845, "loss/policy_avg": 0.03886333, "loss/value_avg": 0.50499833, "val/clipfrac_avg": 0.00028935, "policy/entropy_avg": 0.1860075, "val/ratio": 0.98788619, "val/ratio_var": 8.506e-05, "val/num_eos_tokens": 538, "lr": 2.6e-07, "episode": 17888, "epoch": 0.90343434, "global_step/max_steps": "1118/1238", "percentage": "90.31%", "elapsed_time": "13h 26m 48s", "remaining_time": "1h 26m 35s"} +{"eps": 0, "objective/kl": 40.48139191, "objective/entropy": -66.79815674, "objective/non_score_reward": -2.02406979, "objective/rlhf_reward": 13.48374271, "objective/scores": 15.5, "policy/approxkl_avg": 0.00106471, "policy/clipfrac_avg": 0.00504247, "loss/policy_avg": -0.30915409, "loss/value_avg": 0.63108754, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2105546, "val/ratio": 0.98606443, "val/ratio_var": 7.681e-05, "val/num_eos_tokens": 341, "lr": 2.5e-07, "episode": 17904, "epoch": 0.90424242, "global_step/max_steps": "1119/1238", "percentage": "90.39%", "elapsed_time": "13h 27m 21s", "remaining_time": "1h 25m 51s"} +{"eps": 0, "objective/kl": 37.41370392, "objective/entropy": -109.19269562, "objective/non_score_reward": -1.87068546, "objective/rlhf_reward": 15.55900192, "objective/scores": 17.375, "policy/approxkl_avg": 0.00080963, "policy/clipfrac_avg": 0.00700885, "loss/policy_avg": -0.03548686, "loss/value_avg": 0.43115413, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15267754, "val/ratio": 0.9900794, "val/ratio_var": 6.87e-05, "val/num_eos_tokens": 27, "lr": 2.5e-07, "episode": 17920, "epoch": 0.90505051, "global_step/max_steps": "1120/1238", "percentage": "90.47%", "elapsed_time": "13h 27m 50s", "remaining_time": "1h 25m 6s"} +{"eps": 0, "objective/kl": 34.01490021, "objective/entropy": -81.66447449, "objective/non_score_reward": -1.70074499, "objective/rlhf_reward": 14.06488037, "objective/scores": 15.75, "policy/approxkl_avg": 0.00110455, "policy/clipfrac_avg": 0.00497574, "loss/policy_avg": -0.10384487, "loss/value_avg": 0.55550158, "val/clipfrac_avg": 0.00020559, "policy/entropy_avg": 0.19969654, "val/ratio": 0.98610687, "val/ratio_var": 0.00011405, "val/num_eos_tokens": 424, "lr": 2.5e-07, "episode": 17936, "epoch": 0.90585859, "global_step/max_steps": "1121/1238", "percentage": "90.55%", "elapsed_time": "13h 28m 22s", "remaining_time": "1h 24m 22s"} +{"eps": 0, "objective/kl": 36.03460693, "objective/entropy": -51.88709259, "objective/non_score_reward": -1.80173051, "objective/rlhf_reward": 13.28420734, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00078655, "policy/clipfrac_avg": 0.00519003, "loss/policy_avg": 0.08499134, "loss/value_avg": 0.48207605, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16442871, "val/ratio": 0.98965949, "val/ratio_var": 3.428e-05, "val/num_eos_tokens": 97, "lr": 2.4e-07, "episode": 17952, "epoch": 0.90666667, "global_step/max_steps": "1122/1238", "percentage": "90.63%", "elapsed_time": "13h 28m 54s", "remaining_time": "1h 23m 37s"} +{"eps": 0, "objective/kl": 43.59098816, "objective/entropy": -86.18510437, "objective/non_score_reward": -2.17954922, "objective/rlhf_reward": 14.42201328, "objective/scores": 16.625, "policy/approxkl_avg": 0.00097174, "policy/clipfrac_avg": 0.00642315, "loss/policy_avg": -0.17406422, "loss/value_avg": 0.58962673, "val/clipfrac_avg": 0.00036337, "policy/entropy_avg": 0.18558311, "val/ratio": 0.98733884, "val/ratio_var": 6.847e-05, "val/num_eos_tokens": 28, "lr": 2.4e-07, "episode": 17968, "epoch": 0.90747475, "global_step/max_steps": "1123/1238", "percentage": "90.71%", "elapsed_time": "13h 29m 28s", "remaining_time": "1h 22m 53s"} +{"eps": 0, "objective/kl": 46.46793365, "objective/entropy": -15.71459484, "objective/non_score_reward": -2.32339668, "objective/rlhf_reward": 13.95785332, "objective/scores": 16.25, "policy/approxkl_avg": 0.00090078, "policy/clipfrac_avg": 0.00424796, "loss/policy_avg": 0.00332391, "loss/value_avg": 0.47847858, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18125153, "val/ratio": 0.98787427, "val/ratio_var": 4.735e-05, "val/num_eos_tokens": 91, "lr": 2.3e-07, "episode": 17984, "epoch": 0.90828283, "global_step/max_steps": "1124/1238", "percentage": "90.79%", "elapsed_time": "13h 29m 50s", "remaining_time": "1h 22m 8s"} +{"eps": 0, "objective/kl": 28.81537247, "objective/entropy": -78.6295929, "objective/non_score_reward": -1.4407686, "objective/rlhf_reward": 13.7623558, "objective/scores": 15.1875, "policy/approxkl_avg": 0.00079394, "policy/clipfrac_avg": 0.00239334, "loss/policy_avg": -0.34681112, "loss/value_avg": 1.87613487, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1434046, "val/ratio": 0.9906671, "val/ratio_var": 8.599e-05, "val/num_eos_tokens": 162, "lr": 2.3e-07, "episode": 18000, "epoch": 0.90909091, "global_step/max_steps": "1125/1238", "percentage": "90.87%", "elapsed_time": "13h 30m 19s", "remaining_time": "1h 21m 23s"} +{"eps": 0, "objective/kl": 41.48188019, "objective/entropy": -84.69230652, "objective/non_score_reward": -2.07409406, "objective/rlhf_reward": 14.40246773, "objective/scores": 16.5, "policy/approxkl_avg": 0.00075991, "policy/clipfrac_avg": 0.00491856, "loss/policy_avg": -0.10182957, "loss/value_avg": 0.92626697, "val/clipfrac_avg": 0.0003811, "policy/entropy_avg": 0.14319229, "val/ratio": 0.99042088, "val/ratio_var": 5.685e-05, "val/num_eos_tokens": 279, "lr": 2.3e-07, "episode": 18016, "epoch": 0.90989899, "global_step/max_steps": "1126/1238", "percentage": "90.95%", "elapsed_time": "13h 30m 51s", "remaining_time": "1h 20m 39s"} +{"eps": 0, "objective/kl": 36.32238007, "objective/entropy": 6.48763561, "objective/non_score_reward": -1.81611907, "objective/rlhf_reward": 12.40263081, "objective/scores": 14.25, "policy/approxkl_avg": 0.00121173, "policy/clipfrac_avg": 0.00392553, "loss/policy_avg": -0.05897387, "loss/value_avg": 0.65506637, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.22541046, "val/ratio": 0.98407125, "val/ratio_var": 4.843e-05, "val/num_eos_tokens": 48, "lr": 2.2e-07, "episode": 18032, "epoch": 0.91070707, "global_step/max_steps": "1127/1238", "percentage": "91.03%", "elapsed_time": "13h 31m 13s", "remaining_time": "1h 19m 53s"} +{"eps": 0, "objective/kl": 43.22908783, "objective/entropy": -81.51445007, "objective/non_score_reward": -2.16145444, "objective/rlhf_reward": 14.60417175, "objective/scores": 16.75, "policy/approxkl_avg": 0.000822, "policy/clipfrac_avg": 0.00318674, "loss/policy_avg": -0.21189059, "loss/value_avg": 0.27014664, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16611195, "val/ratio": 0.9885214, "val/ratio_var": 5.36e-05, "val/num_eos_tokens": 379, "lr": 2.2e-07, "episode": 18048, "epoch": 0.91151515, "global_step/max_steps": "1128/1238", "percentage": "91.11%", "elapsed_time": "13h 31m 50s", "remaining_time": "1h 19m 10s"} +{"eps": 0, "objective/kl": 43.02030182, "objective/entropy": -90.69831085, "objective/non_score_reward": -2.15101528, "objective/rlhf_reward": 15.84117317, "objective/scores": 18.0, "policy/approxkl_avg": 0.00089692, "policy/clipfrac_avg": 0.00406639, "loss/policy_avg": 0.07222969, "loss/value_avg": 0.13441604, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18462801, "val/ratio": 0.98813254, "val/ratio_var": 5.761e-05, "val/num_eos_tokens": 122, "lr": 2.1e-07, "episode": 18064, "epoch": 0.91232323, "global_step/max_steps": "1129/1238", "percentage": "91.20%", "elapsed_time": "13h 32m 28s", "remaining_time": "1h 18m 26s"} +{"eps": 0, "objective/kl": 39.51776505, "objective/entropy": -97.6414566, "objective/non_score_reward": -1.97588825, "objective/rlhf_reward": 13.84442425, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00086006, "policy/clipfrac_avg": 0.00265611, "loss/policy_avg": 0.03058287, "loss/value_avg": 0.32031867, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16113853, "val/ratio": 0.98838538, "val/ratio_var": 6.297e-05, "val/num_eos_tokens": 730, "lr": 2.1e-07, "episode": 18080, "epoch": 0.91313131, "global_step/max_steps": "1130/1238", "percentage": "91.28%", "elapsed_time": "13h 33m 10s", "remaining_time": "1h 17m 43s"} +{"eps": 0, "objective/kl": 34.70388412, "objective/entropy": -67.81413269, "objective/non_score_reward": -1.73519421, "objective/rlhf_reward": 14.07730579, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00090271, "policy/clipfrac_avg": 0.00461231, "loss/policy_avg": 0.00443975, "loss/value_avg": 0.53376269, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18857193, "val/ratio": 0.98755169, "val/ratio_var": 7.778e-05, "val/num_eos_tokens": 89, "lr": 2.1e-07, "episode": 18096, "epoch": 0.91393939, "global_step/max_steps": "1131/1238", "percentage": "91.36%", "elapsed_time": "13h 33m 57s", "remaining_time": "1h 17m 0s"} +{"eps": 0, "objective/kl": 41.89903259, "objective/entropy": -57.45730591, "objective/non_score_reward": -2.09495163, "objective/rlhf_reward": 15.76442337, "objective/scores": 17.875, "policy/approxkl_avg": 0.00101468, "policy/clipfrac_avg": 0.00542709, "loss/policy_avg": -0.06492081, "loss/value_avg": 0.25733531, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21177673, "val/ratio": 0.98590142, "val/ratio_var": 5.11e-05, "val/num_eos_tokens": 81, "lr": 2e-07, "episode": 18112, "epoch": 0.91474747, "global_step/max_steps": "1132/1238", "percentage": "91.44%", "elapsed_time": "13h 34m 28s", "remaining_time": "1h 16m 16s"} +{"eps": 0, "objective/kl": 47.30480957, "objective/entropy": -32.12015533, "objective/non_score_reward": -2.36524057, "objective/rlhf_reward": 13.7206974, "objective/scores": 16.125, "policy/approxkl_avg": 0.0010293, "policy/clipfrac_avg": 0.00415786, "loss/policy_avg": -0.03162733, "loss/value_avg": 0.41586339, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20333099, "val/ratio": 0.98641682, "val/ratio_var": 5.427e-05, "val/num_eos_tokens": 51, "lr": 2e-07, "episode": 18128, "epoch": 0.91555556, "global_step/max_steps": "1133/1238", "percentage": "91.52%", "elapsed_time": "13h 35m 7s", "remaining_time": "1h 15m 32s"} +{"eps": 0, "objective/kl": 41.02958679, "objective/entropy": -68.7517395, "objective/non_score_reward": -2.05147934, "objective/rlhf_reward": 13.80789566, "objective/scores": 15.875, "policy/approxkl_avg": 0.00092402, "policy/clipfrac_avg": 0.00337709, "loss/policy_avg": 0.08644348, "loss/value_avg": 0.27462885, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18203735, "val/ratio": 0.98794508, "val/ratio_var": 5.012e-05, "val/num_eos_tokens": 236, "lr": 2e-07, "episode": 18144, "epoch": 0.91636364, "global_step/max_steps": "1134/1238", "percentage": "91.60%", "elapsed_time": "13h 35m 45s", "remaining_time": "1h 14m 48s"} +{"eps": 0, "objective/kl": 38.1743927, "objective/entropy": -23.64558029, "objective/non_score_reward": -1.90871966, "objective/rlhf_reward": 15.47409248, "objective/scores": 17.375, "policy/approxkl_avg": 0.00099418, "policy/clipfrac_avg": 0.00355174, "loss/policy_avg": 0.06688634, "loss/value_avg": 0.48916578, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21059799, "val/ratio": 0.98646462, "val/ratio_var": 6.111e-05, "val/num_eos_tokens": 4, "lr": 1.9e-07, "episode": 18160, "epoch": 0.91717172, "global_step/max_steps": "1135/1238", "percentage": "91.68%", "elapsed_time": "13h 36m 13s", "remaining_time": "1h 14m 4s"} +{"eps": 0, "objective/kl": 43.0865097, "objective/entropy": -43.90042496, "objective/non_score_reward": -2.15432549, "objective/rlhf_reward": 14.97067451, "objective/scores": 17.125, "policy/approxkl_avg": 0.00100218, "policy/clipfrac_avg": 0.00640224, "loss/policy_avg": 0.09911378, "loss/value_avg": 0.18598923, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19201088, "val/ratio": 0.98684049, "val/ratio_var": 3.593e-05, "val/num_eos_tokens": 455, "lr": 1.9e-07, "episode": 18176, "epoch": 0.9179798, "global_step/max_steps": "1136/1238", "percentage": "91.76%", "elapsed_time": "13h 36m 50s", "remaining_time": "1h 13m 20s"} +{"eps": 0, "objective/kl": 41.71107483, "objective/entropy": -149.05351257, "objective/non_score_reward": -2.08555365, "objective/rlhf_reward": 15.11757088, "objective/scores": 17.25, "policy/approxkl_avg": 0.0006845, "policy/clipfrac_avg": 0.0048371, "loss/policy_avg": -0.04335241, "loss/value_avg": 0.61056364, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14271402, "val/ratio": 0.99021024, "val/ratio_var": 5.703e-05, "val/num_eos_tokens": 356, "lr": 1.8e-07, "episode": 18192, "epoch": 0.91878788, "global_step/max_steps": "1137/1238", "percentage": "91.84%", "elapsed_time": "13h 37m 39s", "remaining_time": "1h 12m 37s"} +{"eps": 0, "objective/kl": 38.60702515, "objective/entropy": -55.38002396, "objective/non_score_reward": -1.93035126, "objective/rlhf_reward": 14.39777374, "objective/scores": 16.375, "policy/approxkl_avg": 0.00091892, "policy/clipfrac_avg": 0.00387992, "loss/policy_avg": -0.03888161, "loss/value_avg": 0.26667249, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15883827, "val/ratio": 0.98938906, "val/ratio_var": 5.719e-05, "val/num_eos_tokens": 41, "lr": 1.8e-07, "episode": 18208, "epoch": 0.91959596, "global_step/max_steps": "1138/1238", "percentage": "91.92%", "elapsed_time": "13h 38m 9s", "remaining_time": "1h 11m 53s"} +{"eps": 0, "objective/kl": 34.56838989, "objective/entropy": -44.06977463, "objective/non_score_reward": -1.72841954, "objective/rlhf_reward": 14.62314224, "objective/scores": 16.375, "policy/approxkl_avg": 0.00080487, "policy/clipfrac_avg": 0.00636605, "loss/policy_avg": 0.0642922, "loss/value_avg": 0.80669332, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18921661, "val/ratio": 0.9889757, "val/ratio_var": 3.707e-05, "val/num_eos_tokens": 33, "lr": 1.8e-07, "episode": 18224, "epoch": 0.92040404, "global_step/max_steps": "1139/1238", "percentage": "92.00%", "elapsed_time": "13h 38m 35s", "remaining_time": "1h 11m 9s"} +{"eps": 0, "objective/kl": 39.7617569, "objective/entropy": -85.96157074, "objective/non_score_reward": -1.98808777, "objective/rlhf_reward": 14.47284985, "objective/scores": 16.5, "policy/approxkl_avg": 0.00090513, "policy/clipfrac_avg": 0.00309897, "loss/policy_avg": 0.14161836, "loss/value_avg": 0.72871387, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15839195, "val/ratio": 0.98886812, "val/ratio_var": 7.06e-05, "val/num_eos_tokens": 59, "lr": 1.7e-07, "episode": 18240, "epoch": 0.92121212, "global_step/max_steps": "1140/1238", "percentage": "92.08%", "elapsed_time": "13h 39m 30s", "remaining_time": "1h 10m 26s"} +{"eps": 0, "objective/kl": 42.76993561, "objective/entropy": -50.15681458, "objective/non_score_reward": -2.13849688, "objective/rlhf_reward": 14.10369015, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009481, "policy/clipfrac_avg": 0.00518589, "loss/policy_avg": 0.10231739, "loss/value_avg": 0.77106231, "val/clipfrac_avg": 2.878e-05, "policy/entropy_avg": 0.21046543, "val/ratio": 0.9877795, "val/ratio_var": 6.829e-05, "val/num_eos_tokens": 158, "lr": 1.7e-07, "episode": 18256, "epoch": 0.9220202, "global_step/max_steps": "1141/1238", "percentage": "92.16%", "elapsed_time": "13h 40m 7s", "remaining_time": "1h 9m 43s"} +{"eps": 0, "objective/kl": 37.26749802, "objective/entropy": -83.49034119, "objective/non_score_reward": -1.86337495, "objective/rlhf_reward": 13.49600029, "objective/scores": 15.375, "policy/approxkl_avg": 0.00082033, "policy/clipfrac_avg": 0.00706325, "loss/policy_avg": 0.06746611, "loss/value_avg": 0.32568371, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16310883, "val/ratio": 0.98980141, "val/ratio_var": 4.546e-05, "val/num_eos_tokens": 109, "lr": 1.7e-07, "episode": 18272, "epoch": 0.92282828, "global_step/max_steps": "1142/1238", "percentage": "92.25%", "elapsed_time": "13h 40m 42s", "remaining_time": "1h 8m 59s"} +{"eps": 0, "objective/kl": 45.40270233, "objective/entropy": -22.76852226, "objective/non_score_reward": -2.27013516, "objective/rlhf_reward": 14.22205162, "objective/scores": 16.5, "policy/approxkl_avg": 0.00109603, "policy/clipfrac_avg": 0.0048814, "loss/policy_avg": -0.17026603, "loss/value_avg": 0.35972562, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21602249, "val/ratio": 0.98551977, "val/ratio_var": 3.711e-05, "val/num_eos_tokens": 105, "lr": 1.6e-07, "episode": 18288, "epoch": 0.92363636, "global_step/max_steps": "1143/1238", "percentage": "92.33%", "elapsed_time": "13h 41m 21s", "remaining_time": "1h 8m 15s"} +{"eps": 0, "objective/kl": 33.05713272, "objective/entropy": -49.97838593, "objective/non_score_reward": -1.65285683, "objective/rlhf_reward": 15.80026817, "objective/scores": 17.5, "policy/approxkl_avg": 0.00096398, "policy/clipfrac_avg": 0.00445273, "loss/policy_avg": 0.19920386, "loss/value_avg": 0.53002131, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18081665, "val/ratio": 0.98737025, "val/ratio_var": 4.911e-05, "val/num_eos_tokens": 508, "lr": 1.6e-07, "episode": 18304, "epoch": 0.92444444, "global_step/max_steps": "1144/1238", "percentage": "92.41%", "elapsed_time": "13h 41m 49s", "remaining_time": "1h 7m 31s"} +{"eps": 0, "objective/kl": 46.23842239, "objective/entropy": -32.5231781, "objective/non_score_reward": -2.31192112, "objective/rlhf_reward": 14.33651733, "objective/scores": 16.625, "policy/approxkl_avg": 0.00099815, "policy/clipfrac_avg": 0.00296098, "loss/policy_avg": -0.09338353, "loss/value_avg": 0.23753159, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20490646, "val/ratio": 0.98616403, "val/ratio_var": 5.283e-05, "val/num_eos_tokens": 284, "lr": 1.6e-07, "episode": 18320, "epoch": 0.92525253, "global_step/max_steps": "1145/1238", "percentage": "92.49%", "elapsed_time": "13h 42m 47s", "remaining_time": "1h 6m 49s"} +{"eps": 0, "objective/kl": 31.83081627, "objective/entropy": -102.07046509, "objective/non_score_reward": -1.59154093, "objective/rlhf_reward": 14.75220871, "objective/scores": 16.375, "policy/approxkl_avg": 0.00089433, "policy/clipfrac_avg": 0.00694577, "loss/policy_avg": -0.19417951, "loss/value_avg": 0.83059359, "val/clipfrac_avg": 0.00017361, "policy/entropy_avg": 0.16139507, "val/ratio": 0.98839062, "val/ratio_var": 7.157e-05, "val/num_eos_tokens": 780, "lr": 1.5e-07, "episode": 18336, "epoch": 0.92606061, "global_step/max_steps": "1146/1238", "percentage": "92.57%", "elapsed_time": "13h 43m 28s", "remaining_time": "1h 6m 6s"} +{"eps": 0, "objective/kl": 31.00863266, "objective/entropy": -73.28153992, "objective/non_score_reward": -1.55043173, "objective/rlhf_reward": 13.94175529, "objective/scores": 15.5, "policy/approxkl_avg": 0.00082648, "policy/clipfrac_avg": 0.00495097, "loss/policy_avg": -0.14431903, "loss/value_avg": 1.09122777, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15498734, "val/ratio": 0.98895973, "val/ratio_var": 6.788e-05, "val/num_eos_tokens": 441, "lr": 1.5e-07, "episode": 18352, "epoch": 0.92686869, "global_step/max_steps": "1147/1238", "percentage": "92.65%", "elapsed_time": "13h 44m 4s", "remaining_time": "1h 5m 22s"} +{"eps": 0, "objective/kl": 37.21107101, "objective/entropy": -56.73724747, "objective/non_score_reward": -1.8605535, "objective/rlhf_reward": 13.06132126, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00093012, "policy/clipfrac_avg": 0.00376205, "loss/policy_avg": -0.03664657, "loss/value_avg": 0.54893726, "val/clipfrac_avg": 0.01052903, "policy/entropy_avg": 0.16833878, "val/ratio": 0.98845148, "val/ratio_var": 3.902e-05, "val/num_eos_tokens": 126, "lr": 1.5e-07, "episode": 18368, "epoch": 0.92767677, "global_step/max_steps": "1148/1238", "percentage": "92.73%", "elapsed_time": "13h 44m 44s", "remaining_time": "1h 4m 39s"} +{"eps": 0, "objective/kl": 41.92686462, "objective/entropy": -37.70539856, "objective/non_score_reward": -2.09634328, "objective/rlhf_reward": 14.62240696, "objective/scores": 16.75, "policy/approxkl_avg": 0.00109442, "policy/clipfrac_avg": 0.00474914, "loss/policy_avg": -0.13550133, "loss/value_avg": 0.31003577, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21954727, "val/ratio": 0.98519343, "val/ratio_var": 6.72e-05, "val/num_eos_tokens": 114, "lr": 1.4e-07, "episode": 18384, "epoch": 0.92848485, "global_step/max_steps": "1149/1238", "percentage": "92.81%", "elapsed_time": "13h 45m 18s", "remaining_time": "1h 3m 55s"} +{"eps": 0, "objective/kl": 38.18930054, "objective/entropy": -55.67281723, "objective/non_score_reward": -1.90946507, "objective/rlhf_reward": 14.02803421, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00101141, "policy/clipfrac_avg": 0.0053071, "loss/policy_avg": -0.06519679, "loss/value_avg": 0.49521238, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20238876, "val/ratio": 0.98630631, "val/ratio_var": 0.00010382, "val/num_eos_tokens": 103, "lr": 1.4e-07, "episode": 18400, "epoch": 0.92929293, "global_step/max_steps": "1150/1238", "percentage": "92.89%", "elapsed_time": "13h 45m 55s", "remaining_time": "1h 3m 12s"} +{"eps": 0, "objective/kl": 39.29536057, "objective/entropy": -64.18814087, "objective/non_score_reward": -1.96476793, "objective/rlhf_reward": 13.74616909, "objective/scores": 15.6875, "policy/approxkl_avg": 0.00090696, "policy/clipfrac_avg": 0.00267383, "loss/policy_avg": 0.02585557, "loss/value_avg": 0.77427, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19156981, "val/ratio": 0.98850405, "val/ratio_var": 7.799e-05, "val/num_eos_tokens": 21, "lr": 1.4e-07, "episode": 18416, "epoch": 0.93010101, "global_step/max_steps": "1151/1238", "percentage": "92.97%", "elapsed_time": "13h 46m 29s", "remaining_time": "1h 2m 28s"} +{"eps": 0, "objective/kl": 40.6189003, "objective/entropy": -59.23526764, "objective/non_score_reward": -2.03094506, "objective/rlhf_reward": 11.54718018, "objective/scores": 13.5625, "policy/approxkl_avg": 0.00088816, "policy/clipfrac_avg": 0.00322373, "loss/policy_avg": 0.01431315, "loss/value_avg": 0.62860578, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17874622, "val/ratio": 0.98873734, "val/ratio_var": 5.043e-05, "val/num_eos_tokens": 353, "lr": 1.3e-07, "episode": 18432, "epoch": 0.93090909, "global_step/max_steps": "1152/1238", "percentage": "93.05%", "elapsed_time": "13h 47m 2s", "remaining_time": "1h 1m 44s"} +{"eps": 0, "objective/kl": 46.32222366, "objective/entropy": -56.3327713, "objective/non_score_reward": -2.31611109, "objective/rlhf_reward": 15.99638939, "objective/scores": 18.25, "policy/approxkl_avg": 0.00092886, "policy/clipfrac_avg": 0.00274551, "loss/policy_avg": -0.08571564, "loss/value_avg": 0.24318826, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1957531, "val/ratio": 0.98694265, "val/ratio_var": 4.259e-05, "val/num_eos_tokens": 26, "lr": 1.3e-07, "episode": 18448, "epoch": 0.93171717, "global_step/max_steps": "1153/1238", "percentage": "93.13%", "elapsed_time": "13h 47m 53s", "remaining_time": "1h 1m 1s"} +{"eps": 0, "objective/kl": 33.21459198, "objective/entropy": -81.86225891, "objective/non_score_reward": -1.66072965, "objective/rlhf_reward": 13.62833309, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00088942, "policy/clipfrac_avg": 0.00322179, "loss/policy_avg": -0.09154314, "loss/value_avg": 0.58333594, "val/clipfrac_avg": 0.00050912, "policy/entropy_avg": 0.16341472, "val/ratio": 0.98800576, "val/ratio_var": 7.092e-05, "val/num_eos_tokens": 503, "lr": 1.3e-07, "episode": 18464, "epoch": 0.93252525, "global_step/max_steps": "1154/1238", "percentage": "93.21%", "elapsed_time": "13h 48m 29s", "remaining_time": "1h 0m 18s"} +{"eps": 0, "objective/kl": 37.38941193, "objective/entropy": -86.53608704, "objective/non_score_reward": -1.8694706, "objective/rlhf_reward": 14.5914669, "objective/scores": 16.5, "policy/approxkl_avg": 0.00103676, "policy/clipfrac_avg": 0.00769103, "loss/policy_avg": 0.10637004, "loss/value_avg": 0.54882717, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16927147, "val/ratio": 0.98814118, "val/ratio_var": 8.204e-05, "val/num_eos_tokens": 399, "lr": 1.3e-07, "episode": 18480, "epoch": 0.93333333, "global_step/max_steps": "1155/1238", "percentage": "93.30%", "elapsed_time": "13h 48m 58s", "remaining_time": "59m 34s"} +{"eps": 0, "objective/kl": 44.40808105, "objective/entropy": -87.75643921, "objective/non_score_reward": -2.22040415, "objective/rlhf_reward": 14.40459633, "objective/scores": 16.625, "policy/approxkl_avg": 0.00085445, "policy/clipfrac_avg": 0.0035027, "loss/policy_avg": -0.02412903, "loss/value_avg": 0.42145044, "val/clipfrac_avg": 9.058e-05, "policy/entropy_avg": 0.16388321, "val/ratio": 0.98899311, "val/ratio_var": 6.747e-05, "val/num_eos_tokens": 8, "lr": 1.2e-07, "episode": 18496, "epoch": 0.93414141, "global_step/max_steps": "1156/1238", "percentage": "93.38%", "elapsed_time": "13h 49m 34s", "remaining_time": "58m 50s"} +{"eps": 0, "objective/kl": 39.9620285, "objective/entropy": -90.10191345, "objective/non_score_reward": -1.99810147, "objective/rlhf_reward": 16.14252472, "objective/scores": 18.125, "policy/approxkl_avg": 0.00085466, "policy/clipfrac_avg": 0.00538104, "loss/policy_avg": -0.08957675, "loss/value_avg": 0.33554566, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16065407, "val/ratio": 0.98916519, "val/ratio_var": 4.784e-05, "val/num_eos_tokens": 3, "lr": 1.2e-07, "episode": 18512, "epoch": 0.93494949, "global_step/max_steps": "1157/1238", "percentage": "93.46%", "elapsed_time": "13h 50m 8s", "remaining_time": "58m 6s"} +{"eps": 0, "objective/kl": 40.30827332, "objective/entropy": -63.45883179, "objective/non_score_reward": -2.01541376, "objective/rlhf_reward": 13.63302422, "objective/scores": 15.625, "policy/approxkl_avg": 0.00086526, "policy/clipfrac_avg": 0.00967923, "loss/policy_avg": -0.0647333, "loss/value_avg": 0.42259383, "val/clipfrac_avg": 0.00011748, "policy/entropy_avg": 0.16831875, "val/ratio": 0.98950088, "val/ratio_var": 3.429e-05, "val/num_eos_tokens": 301, "lr": 1.2e-07, "episode": 18528, "epoch": 0.93575758, "global_step/max_steps": "1158/1238", "percentage": "93.54%", "elapsed_time": "13h 50m 49s", "remaining_time": "57m 23s"} +{"eps": 0, "objective/kl": 36.45595169, "objective/entropy": -65.75634766, "objective/non_score_reward": -1.82279766, "objective/rlhf_reward": 13.13813972, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00084741, "policy/clipfrac_avg": 0.00681584, "loss/policy_avg": 0.19546494, "loss/value_avg": 0.69054794, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16717529, "val/ratio": 0.98945463, "val/ratio_var": 7.424e-05, "val/num_eos_tokens": 124, "lr": 1.1e-07, "episode": 18544, "epoch": 0.93656566, "global_step/max_steps": "1159/1238", "percentage": "93.62%", "elapsed_time": "13h 51m 38s", "remaining_time": "56m 41s"} +{"eps": 0, "objective/kl": 37.05547714, "objective/entropy": -168.40960693, "objective/non_score_reward": -1.85277367, "objective/rlhf_reward": 14.08472633, "objective/scores": 15.9375, "policy/approxkl_avg": 0.00079128, "policy/clipfrac_avg": 0.00361628, "loss/policy_avg": 0.10427418, "loss/value_avg": 0.71158254, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15630913, "val/ratio": 0.99012148, "val/ratio_var": 7.213e-05, "val/num_eos_tokens": 427, "lr": 1.1e-07, "episode": 18560, "epoch": 0.93737374, "global_step/max_steps": "1160/1238", "percentage": "93.70%", "elapsed_time": "13h 52m 22s", "remaining_time": "55m 58s"} +{"eps": 0, "objective/kl": 44.20101929, "objective/entropy": -70.2331543, "objective/non_score_reward": -2.21005082, "objective/rlhf_reward": 16.10244942, "objective/scores": 18.25, "policy/approxkl_avg": 0.00114082, "policy/clipfrac_avg": 0.00494441, "loss/policy_avg": -0.12785541, "loss/value_avg": 0.39417845, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19538498, "val/ratio": 0.98658842, "val/ratio_var": 6.051e-05, "val/num_eos_tokens": 331, "lr": 1.1e-07, "episode": 18576, "epoch": 0.93818182, "global_step/max_steps": "1161/1238", "percentage": "93.78%", "elapsed_time": "13h 53m 7s", "remaining_time": "55m 15s"} +{"eps": 0, "objective/kl": 43.31715012, "objective/entropy": -37.64844131, "objective/non_score_reward": -2.16585732, "objective/rlhf_reward": 13.09976768, "objective/scores": 15.25, "policy/approxkl_avg": 0.00099002, "policy/clipfrac_avg": 0.00300791, "loss/policy_avg": -0.03844345, "loss/value_avg": 0.65571821, "val/clipfrac_avg": 0.00039063, "policy/entropy_avg": 0.20216942, "val/ratio": 0.9874059, "val/ratio_var": 6.271e-05, "val/num_eos_tokens": 199, "lr": 1.1e-07, "episode": 18592, "epoch": 0.9389899, "global_step/max_steps": "1162/1238", "percentage": "93.86%", "elapsed_time": "13h 53m 38s", "remaining_time": "54m 31s"} +{"eps": 0, "objective/kl": 38.83243942, "objective/entropy": -83.78005981, "objective/non_score_reward": -1.94162202, "objective/rlhf_reward": 15.86306477, "objective/scores": 17.75, "policy/approxkl_avg": 0.00086891, "policy/clipfrac_avg": 0.00458956, "loss/policy_avg": 0.05490084, "loss/value_avg": 0.44468832, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17558479, "val/ratio": 0.98784733, "val/ratio_var": 5.755e-05, "val/num_eos_tokens": 209, "lr": 1e-07, "episode": 18608, "epoch": 0.93979798, "global_step/max_steps": "1163/1238", "percentage": "93.94%", "elapsed_time": "13h 54m 8s", "remaining_time": "53m 47s"} +{"eps": 0, "objective/kl": 28.18227386, "objective/entropy": -106.71968079, "objective/non_score_reward": -1.40911365, "objective/rlhf_reward": 13.05182457, "objective/scores": 14.4375, "policy/approxkl_avg": 0.00066044, "policy/clipfrac_avg": 0.00607199, "loss/policy_avg": 0.56522292, "loss/value_avg": 4.45482779, "val/clipfrac_avg": 0.00320513, "policy/entropy_avg": 0.13239336, "val/ratio": 0.99106872, "val/ratio_var": 5.329e-05, "val/num_eos_tokens": 199, "lr": 1e-07, "episode": 18624, "epoch": 0.94060606, "global_step/max_steps": "1164/1238", "percentage": "94.02%", "elapsed_time": "13h 54m 44s", "remaining_time": "53m 4s"} +{"eps": 0, "objective/kl": 46.05052948, "objective/entropy": -50.39229965, "objective/non_score_reward": -2.30252647, "objective/rlhf_reward": 15.77559853, "objective/scores": 18.125, "policy/approxkl_avg": 0.0010014, "policy/clipfrac_avg": 0.00413426, "loss/policy_avg": 0.03055404, "loss/value_avg": 0.38135013, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20552826, "val/ratio": 0.98620898, "val/ratio_var": 5.763e-05, "val/num_eos_tokens": 147, "lr": 1e-07, "episode": 18640, "epoch": 0.94141414, "global_step/max_steps": "1165/1238", "percentage": "94.10%", "elapsed_time": "13h 55m 17s", "remaining_time": "52m 20s"} +{"eps": 0, "objective/kl": 46.88139343, "objective/entropy": -48.58641434, "objective/non_score_reward": -2.34406972, "objective/rlhf_reward": 14.74967957, "objective/scores": 17.125, "policy/approxkl_avg": 0.00095884, "policy/clipfrac_avg": 0.00315808, "loss/policy_avg": 0.01442549, "loss/value_avg": 0.17465442, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19722748, "val/ratio": 0.98723626, "val/ratio_var": 5.964e-05, "val/num_eos_tokens": 150, "lr": 9e-08, "episode": 18656, "epoch": 0.94222222, "global_step/max_steps": "1166/1238", "percentage": "94.18%", "elapsed_time": "13h 55m 47s", "remaining_time": "51m 36s"} +{"eps": 0, "objective/kl": 38.01190948, "objective/entropy": -76.38983154, "objective/non_score_reward": -1.90059543, "objective/rlhf_reward": 11.91971684, "objective/scores": 13.8125, "policy/approxkl_avg": 0.0007047, "policy/clipfrac_avg": 0.00289586, "loss/policy_avg": 0.12084536, "loss/value_avg": 0.58313978, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15113831, "val/ratio": 0.99073488, "val/ratio_var": 6.114e-05, "val/num_eos_tokens": 417, "lr": 9e-08, "episode": 18672, "epoch": 0.9430303, "global_step/max_steps": "1167/1238", "percentage": "94.26%", "elapsed_time": "13h 56m 22s", "remaining_time": "50m 53s"} +{"eps": 0, "objective/kl": 44.53829956, "objective/entropy": -190.17677307, "objective/non_score_reward": -2.22691512, "objective/rlhf_reward": 14.88246059, "objective/scores": 17.125, "policy/approxkl_avg": 0.00074118, "policy/clipfrac_avg": 0.0051434, "loss/policy_avg": -0.17211503, "loss/value_avg": 0.29965419, "val/clipfrac_avg": 1.347e-05, "policy/entropy_avg": 0.14809704, "val/ratio": 0.99010354, "val/ratio_var": 4.8e-05, "val/num_eos_tokens": 76, "lr": 9e-08, "episode": 18688, "epoch": 0.94383838, "global_step/max_steps": "1168/1238", "percentage": "94.35%", "elapsed_time": "13h 57m 24s", "remaining_time": "50m 11s"} +{"eps": 0, "objective/kl": 43.76121521, "objective/entropy": -26.35421181, "objective/non_score_reward": -2.18806052, "objective/rlhf_reward": 12.90568924, "objective/scores": 15.125, "policy/approxkl_avg": 0.00101394, "policy/clipfrac_avg": 0.00270761, "loss/policy_avg": -0.04706334, "loss/value_avg": 0.4846586, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19493675, "val/ratio": 0.98696828, "val/ratio_var": 4.459e-05, "val/num_eos_tokens": 515, "lr": 9e-08, "episode": 18704, "epoch": 0.94464646, "global_step/max_steps": "1169/1238", "percentage": "94.43%", "elapsed_time": "13h 58m 9s", "remaining_time": "49m 28s"} +{"eps": 0, "objective/kl": 33.78557205, "objective/entropy": -118.44176483, "objective/non_score_reward": -1.6892786, "objective/rlhf_reward": 14.4669714, "objective/scores": 16.125, "policy/approxkl_avg": 0.00080259, "policy/clipfrac_avg": 0.00503463, "loss/policy_avg": -0.11434232, "loss/value_avg": 0.32796037, "val/clipfrac_avg": 0.00024038, "policy/entropy_avg": 0.17252541, "val/ratio": 0.98927528, "val/ratio_var": 8.976e-05, "val/num_eos_tokens": 534, "lr": 8e-08, "episode": 18720, "epoch": 0.94545455, "global_step/max_steps": "1170/1238", "percentage": "94.51%", "elapsed_time": "13h 58m 53s", "remaining_time": "48m 45s"} +{"eps": 0, "objective/kl": 35.37781525, "objective/entropy": -90.45104218, "objective/non_score_reward": -1.76889086, "objective/rlhf_reward": 15.84048367, "objective/scores": 17.625, "policy/approxkl_avg": 0.00075324, "policy/clipfrac_avg": 0.00782379, "loss/policy_avg": -0.26330432, "loss/value_avg": 0.43789923, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14811134, "val/ratio": 0.99012482, "val/ratio_var": 4.504e-05, "val/num_eos_tokens": 449, "lr": 8e-08, "episode": 18736, "epoch": 0.94626263, "global_step/max_steps": "1171/1238", "percentage": "94.59%", "elapsed_time": "13h 59m 30s", "remaining_time": "48m 1s"} +{"eps": 0, "objective/kl": 45.1164093, "objective/entropy": -90.80900574, "objective/non_score_reward": -2.25582027, "objective/rlhf_reward": 13.74417877, "objective/scores": 16.0, "policy/approxkl_avg": 0.00091958, "policy/clipfrac_avg": 0.00517287, "loss/policy_avg": 0.09692055, "loss/value_avg": 0.25977728, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17873764, "val/ratio": 0.98802269, "val/ratio_var": 5.285e-05, "val/num_eos_tokens": 400, "lr": 8e-08, "episode": 18752, "epoch": 0.94707071, "global_step/max_steps": "1172/1238", "percentage": "94.67%", "elapsed_time": "14h 0m 13s", "remaining_time": "47m 18s"} +{"eps": 0, "objective/kl": 37.62765503, "objective/entropy": -163.6809845, "objective/non_score_reward": -1.88138282, "objective/rlhf_reward": 14.59518051, "objective/scores": 16.5, "policy/approxkl_avg": 0.00078813, "policy/clipfrac_avg": 0.00375298, "loss/policy_avg": 0.09063289, "loss/value_avg": 0.57539827, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15079403, "val/ratio": 0.98916912, "val/ratio_var": 4.977e-05, "val/num_eos_tokens": 262, "lr": 8e-08, "episode": 18768, "epoch": 0.94787879, "global_step/max_steps": "1173/1238", "percentage": "94.75%", "elapsed_time": "14h 1m 41s", "remaining_time": "46m 38s"} +{"eps": 0, "objective/kl": 38.04013062, "objective/entropy": -171.24186707, "objective/non_score_reward": -1.90200651, "objective/rlhf_reward": 13.17611885, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00066566, "policy/clipfrac_avg": 0.00337702, "loss/policy_avg": 0.14138484, "loss/value_avg": 0.90886903, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14909267, "val/ratio": 0.9910388, "val/ratio_var": 5.572e-05, "val/num_eos_tokens": 584, "lr": 8e-08, "episode": 18784, "epoch": 0.94868687, "global_step/max_steps": "1174/1238", "percentage": "94.83%", "elapsed_time": "14h 2m 39s", "remaining_time": "45m 56s"} +{"eps": 0, "objective/kl": 43.64716339, "objective/entropy": -50.93875885, "objective/non_score_reward": -2.18235826, "objective/rlhf_reward": 13.80201721, "objective/scores": 16.0, "policy/approxkl_avg": 0.00099526, "policy/clipfrac_avg": 0.00277158, "loss/policy_avg": -0.091682, "loss/value_avg": 0.44036585, "val/clipfrac_avg": 0.00041118, "policy/entropy_avg": 0.17894936, "val/ratio": 0.98708868, "val/ratio_var": 6.252e-05, "val/num_eos_tokens": 169, "lr": 7e-08, "episode": 18800, "epoch": 0.94949495, "global_step/max_steps": "1175/1238", "percentage": "94.91%", "elapsed_time": "14h 3m 20s", "remaining_time": "45m 13s"} +{"eps": 0, "objective/kl": 43.21948624, "objective/entropy": -153.31974792, "objective/non_score_reward": -2.16097426, "objective/rlhf_reward": 12.74136925, "objective/scores": 14.875, "policy/approxkl_avg": 0.00091148, "policy/clipfrac_avg": 0.00419952, "loss/policy_avg": -0.17822237, "loss/value_avg": 0.43820316, "val/clipfrac_avg": 3.815e-05, "policy/entropy_avg": 0.18891335, "val/ratio": 0.98687488, "val/ratio_var": 7.712e-05, "val/num_eos_tokens": 23, "lr": 7e-08, "episode": 18816, "epoch": 0.95030303, "global_step/max_steps": "1176/1238", "percentage": "94.99%", "elapsed_time": "14h 5m 11s", "remaining_time": "44m 33s"} +{"eps": 0, "objective/kl": 46.93393707, "objective/entropy": -62.70560074, "objective/non_score_reward": -2.34669685, "objective/rlhf_reward": 15.28611565, "objective/scores": 17.625, "policy/approxkl_avg": 0.00092566, "policy/clipfrac_avg": 0.00534579, "loss/policy_avg": -0.29926538, "loss/value_avg": 0.24289565, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19078445, "val/ratio": 0.98789066, "val/ratio_var": 5.454e-05, "val/num_eos_tokens": 267, "lr": 7e-08, "episode": 18832, "epoch": 0.95111111, "global_step/max_steps": "1177/1238", "percentage": "95.07%", "elapsed_time": "14h 5m 53s", "remaining_time": "43m 50s"} +{"eps": 0, "objective/kl": 38.56993103, "objective/entropy": -102.23303223, "objective/non_score_reward": -1.92849672, "objective/rlhf_reward": 13.35275269, "objective/scores": 15.25, "policy/approxkl_avg": 0.00081283, "policy/clipfrac_avg": 0.00465432, "loss/policy_avg": -0.07624176, "loss/value_avg": 1.02765357, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16016388, "val/ratio": 0.9896093, "val/ratio_var": 5.182e-05, "val/num_eos_tokens": 33, "lr": 7e-08, "episode": 18848, "epoch": 0.95191919, "global_step/max_steps": "1178/1238", "percentage": "95.15%", "elapsed_time": "14h 6m 51s", "remaining_time": "43m 8s"} +{"eps": 0, "objective/kl": 39.0270462, "objective/entropy": -59.69445038, "objective/non_score_reward": -1.95135236, "objective/rlhf_reward": 13.17364693, "objective/scores": 15.125, "policy/approxkl_avg": 0.00103909, "policy/clipfrac_avg": 0.00358544, "loss/policy_avg": 0.03581414, "loss/value_avg": 0.57642978, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19232655, "val/ratio": 0.98689425, "val/ratio_var": 6.958e-05, "val/num_eos_tokens": 192, "lr": 6e-08, "episode": 18864, "epoch": 0.95272727, "global_step/max_steps": "1179/1238", "percentage": "95.23%", "elapsed_time": "14h 7m 22s", "remaining_time": "42m 24s"} +{"eps": 0, "objective/kl": 47.32844543, "objective/entropy": -40.50076294, "objective/non_score_reward": -2.36642218, "objective/rlhf_reward": 13.85232735, "objective/scores": 16.25, "policy/approxkl_avg": 0.00100827, "policy/clipfrac_avg": 0.0057847, "loss/policy_avg": 0.24797323, "loss/value_avg": 0.28458467, "val/clipfrac_avg": 0.00013242, "policy/entropy_avg": 0.19600677, "val/ratio": 0.98692989, "val/ratio_var": 4.634e-05, "val/num_eos_tokens": 504, "lr": 6e-08, "episode": 18880, "epoch": 0.95353535, "global_step/max_steps": "1180/1238", "percentage": "95.32%", "elapsed_time": "14h 8m 0s", "remaining_time": "41m 40s"} +{"eps": 0, "objective/kl": 38.19042587, "objective/entropy": -87.35359192, "objective/non_score_reward": -1.90952134, "objective/rlhf_reward": 12.5748539, "objective/scores": 14.5, "policy/approxkl_avg": 0.00069387, "policy/clipfrac_avg": 0.00227457, "loss/policy_avg": -0.01105659, "loss/value_avg": 0.2119559, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14525604, "val/ratio": 0.99020791, "val/ratio_var": 5.083e-05, "val/num_eos_tokens": 672, "lr": 6e-08, "episode": 18896, "epoch": 0.95434343, "global_step/max_steps": "1181/1238", "percentage": "95.40%", "elapsed_time": "14h 8m 39s", "remaining_time": "40m 57s"} +{"eps": 0, "objective/kl": 36.28763962, "objective/entropy": -131.93319702, "objective/non_score_reward": -1.81438196, "objective/rlhf_reward": 12.09967995, "objective/scores": 13.9375, "policy/approxkl_avg": 0.00069964, "policy/clipfrac_avg": 0.00330938, "loss/policy_avg": 0.56245565, "loss/value_avg": 0.93662083, "val/clipfrac_avg": 0.0014881, "policy/entropy_avg": 0.14076281, "val/ratio": 0.99091542, "val/ratio_var": 5.887e-05, "val/num_eos_tokens": 240, "lr": 6e-08, "episode": 18912, "epoch": 0.95515152, "global_step/max_steps": "1182/1238", "percentage": "95.48%", "elapsed_time": "14h 9m 14s", "remaining_time": "40m 14s"} +{"eps": 0, "objective/kl": 39.66671371, "objective/entropy": -117.65222168, "objective/non_score_reward": -1.98333561, "objective/rlhf_reward": 13.29791451, "objective/scores": 15.25, "policy/approxkl_avg": 0.00079329, "policy/clipfrac_avg": 0.00292909, "loss/policy_avg": -0.19112438, "loss/value_avg": 0.62474775, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16626358, "val/ratio": 0.98935318, "val/ratio_var": 5.723e-05, "val/num_eos_tokens": 41, "lr": 6e-08, "episode": 18928, "epoch": 0.9559596, "global_step/max_steps": "1183/1238", "percentage": "95.56%", "elapsed_time": "14h 10m 41s", "remaining_time": "39m 33s"} +{"eps": 0, "objective/kl": 38.85816956, "objective/entropy": -59.19387054, "objective/non_score_reward": -1.94290853, "objective/rlhf_reward": 12.28365421, "objective/scores": 14.25, "policy/approxkl_avg": 0.00090588, "policy/clipfrac_avg": 0.00439071, "loss/policy_avg": -0.01887227, "loss/value_avg": 0.68143642, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1727705, "val/ratio": 0.98888069, "val/ratio_var": 5.87e-05, "val/num_eos_tokens": 391, "lr": 5e-08, "episode": 18944, "epoch": 0.95676768, "global_step/max_steps": "1184/1238", "percentage": "95.64%", "elapsed_time": "14h 11m 12s", "remaining_time": "38m 49s"} +{"eps": 0, "objective/kl": 35.59428406, "objective/entropy": -72.95974731, "objective/non_score_reward": -1.77971435, "objective/rlhf_reward": 13.67341042, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00082016, "policy/clipfrac_avg": 0.00414535, "loss/policy_avg": -0.12193583, "loss/value_avg": 0.40720278, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17438912, "val/ratio": 0.98887366, "val/ratio_var": 6.133e-05, "val/num_eos_tokens": 60, "lr": 5e-08, "episode": 18960, "epoch": 0.95757576, "global_step/max_steps": "1185/1238", "percentage": "95.72%", "elapsed_time": "14h 11m 53s", "remaining_time": "38m 6s"} +{"eps": 0, "objective/kl": 41.09333038, "objective/entropy": -97.86048889, "objective/non_score_reward": -2.05466652, "objective/rlhf_reward": 13.71095943, "objective/scores": 15.75, "policy/approxkl_avg": 0.00078003, "policy/clipfrac_avg": 0.00740249, "loss/policy_avg": 0.34098327, "loss/value_avg": 2.11788392, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19989014, "val/ratio": 0.98903447, "val/ratio_var": 7.444e-05, "val/num_eos_tokens": 76, "lr": 5e-08, "episode": 18976, "epoch": 0.95838384, "global_step/max_steps": "1186/1238", "percentage": "95.80%", "elapsed_time": "14h 12m 33s", "remaining_time": "37m 22s"} +{"eps": 0, "objective/kl": 36.85295486, "objective/entropy": -31.46831512, "objective/non_score_reward": -1.84264767, "objective/rlhf_reward": 14.92297745, "objective/scores": 16.75, "policy/approxkl_avg": 0.00120873, "policy/clipfrac_avg": 0.00455514, "loss/policy_avg": -0.16587065, "loss/value_avg": 0.58586359, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20450401, "val/ratio": 0.98743653, "val/ratio_var": 5.31e-05, "val/num_eos_tokens": 113, "lr": 5e-08, "episode": 18992, "epoch": 0.95919192, "global_step/max_steps": "1187/1238", "percentage": "95.88%", "elapsed_time": "14h 12m 56s", "remaining_time": "36m 38s"} +{"eps": 0, "objective/kl": 32.9462738, "objective/entropy": -105.82904816, "objective/non_score_reward": -1.64731371, "objective/rlhf_reward": 15.29018593, "objective/scores": 17.0, "policy/approxkl_avg": 0.00066283, "policy/clipfrac_avg": 0.00486547, "loss/policy_avg": 0.06852813, "loss/value_avg": 0.32328564, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.12561321, "val/ratio": 0.99172378, "val/ratio_var": 3.859e-05, "val/num_eos_tokens": 202, "lr": 5e-08, "episode": 19008, "epoch": 0.96, "global_step/max_steps": "1188/1238", "percentage": "95.96%", "elapsed_time": "14h 13m 47s", "remaining_time": "35m 56s"} +{"eps": 0, "objective/kl": 45.81948471, "objective/entropy": -33.98373032, "objective/non_score_reward": -2.29097438, "objective/rlhf_reward": 13.00590038, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00101973, "policy/clipfrac_avg": 0.00338325, "loss/policy_avg": 0.0203032, "loss/value_avg": 0.43305194, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21395493, "val/ratio": 0.98518384, "val/ratio_var": 5.527e-05, "val/num_eos_tokens": 86, "lr": 4e-08, "episode": 19024, "epoch": 0.96080808, "global_step/max_steps": "1189/1238", "percentage": "96.04%", "elapsed_time": "14h 14m 26s", "remaining_time": "35m 12s"} +{"eps": 0, "objective/kl": 38.25222015, "objective/entropy": -152.57052612, "objective/non_score_reward": -1.91261125, "objective/rlhf_reward": 14.89988899, "objective/scores": 16.75, "policy/approxkl_avg": 0.00076381, "policy/clipfrac_avg": 0.00355914, "loss/policy_avg": -0.61331481, "loss/value_avg": 0.38760403, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16634774, "val/ratio": 0.98929703, "val/ratio_var": 6.732e-05, "val/num_eos_tokens": 392, "lr": 4e-08, "episode": 19040, "epoch": 0.96161616, "global_step/max_steps": "1190/1238", "percentage": "96.12%", "elapsed_time": "14h 15m 8s", "remaining_time": "34m 29s"} +{"eps": 0, "objective/kl": 46.12099457, "objective/entropy": -81.75244141, "objective/non_score_reward": -2.30604982, "objective/rlhf_reward": 13.5376997, "objective/scores": 15.875, "policy/approxkl_avg": 0.00093232, "policy/clipfrac_avg": 0.00327398, "loss/policy_avg": -0.16719648, "loss/value_avg": 0.49441624, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1960032, "val/ratio": 0.98763907, "val/ratio_var": 4.037e-05, "val/num_eos_tokens": 291, "lr": 4e-08, "episode": 19056, "epoch": 0.96242424, "global_step/max_steps": "1191/1238", "percentage": "96.20%", "elapsed_time": "14h 15m 50s", "remaining_time": "33m 46s"} +{"eps": 0, "objective/kl": 38.58598328, "objective/entropy": -58.23650742, "objective/non_score_reward": -1.92929935, "objective/rlhf_reward": 13.14882565, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00106418, "policy/clipfrac_avg": 0.00402071, "loss/policy_avg": 0.08641588, "loss/value_avg": 0.60216951, "val/clipfrac_avg": 0.00021151, "policy/entropy_avg": 0.19188881, "val/ratio": 0.98750436, "val/ratio_var": 7.919e-05, "val/num_eos_tokens": 665, "lr": 4e-08, "episode": 19072, "epoch": 0.96323232, "global_step/max_steps": "1192/1238", "percentage": "96.28%", "elapsed_time": "14h 16m 30s", "remaining_time": "33m 3s"} +{"eps": 0, "objective/kl": 36.08628082, "objective/entropy": -126.37530518, "objective/non_score_reward": -1.80431402, "objective/rlhf_reward": 13.90662384, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008228, "policy/clipfrac_avg": 0.00457361, "loss/policy_avg": 0.09802607, "loss/value_avg": 1.05405283, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15534449, "val/ratio": 0.98891997, "val/ratio_var": 9.046e-05, "val/num_eos_tokens": 325, "lr": 4e-08, "episode": 19088, "epoch": 0.9640404, "global_step/max_steps": "1193/1238", "percentage": "96.37%", "elapsed_time": "14h 17m 5s", "remaining_time": "32m 19s"} +{"eps": 0, "objective/kl": 32.86931992, "objective/entropy": -35.11339569, "objective/non_score_reward": -1.64346611, "objective/rlhf_reward": 14.5518465, "objective/scores": 16.25, "policy/approxkl_avg": 0.00231462, "policy/clipfrac_avg": 0.0031053, "loss/policy_avg": -0.17688647, "loss/value_avg": 0.43707246, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17627335, "val/ratio": 0.98938811, "val/ratio_var": 5.499e-05, "val/num_eos_tokens": 185, "lr": 4e-08, "episode": 19104, "epoch": 0.96484848, "global_step/max_steps": "1194/1238", "percentage": "96.45%", "elapsed_time": "14h 17m 31s", "remaining_time": "31m 36s"} +{"eps": 0, "objective/kl": 40.82053375, "objective/entropy": -92.06233978, "objective/non_score_reward": -2.04102659, "objective/rlhf_reward": 14.51366043, "objective/scores": 16.5, "policy/approxkl_avg": 0.00097471, "policy/clipfrac_avg": 0.0058877, "loss/policy_avg": -0.17504875, "loss/value_avg": 0.20459661, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17911339, "val/ratio": 0.9880842, "val/ratio_var": 4.276e-05, "val/num_eos_tokens": 294, "lr": 3e-08, "episode": 19120, "epoch": 0.96565657, "global_step/max_steps": "1195/1238", "percentage": "96.53%", "elapsed_time": "14h 18m 25s", "remaining_time": "30m 53s"} +{"eps": 0, "objective/kl": 45.62751007, "objective/entropy": -24.15501404, "objective/non_score_reward": -2.28137565, "objective/rlhf_reward": 14.19518661, "objective/scores": 16.5, "policy/approxkl_avg": 0.00103615, "policy/clipfrac_avg": 0.00647178, "loss/policy_avg": -0.12304341, "loss/value_avg": 0.32093346, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19795609, "val/ratio": 0.98683894, "val/ratio_var": 5.582e-05, "val/num_eos_tokens": 209, "lr": 3e-08, "episode": 19136, "epoch": 0.96646465, "global_step/max_steps": "1196/1238", "percentage": "96.61%", "elapsed_time": "14h 18m 52s", "remaining_time": "30m 9s"} +{"eps": 0, "objective/kl": 31.23270988, "objective/entropy": -111.23834229, "objective/non_score_reward": -1.56163549, "objective/rlhf_reward": 13.35242748, "objective/scores": 14.9375, "policy/approxkl_avg": 0.00071766, "policy/clipfrac_avg": 0.00416297, "loss/policy_avg": 0.34430712, "loss/value_avg": 1.53721726, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.24224102, "val/ratio": 0.9907167, "val/ratio_var": 7.462e-05, "val/num_eos_tokens": 524, "lr": 3e-08, "episode": 19152, "epoch": 0.96727273, "global_step/max_steps": "1197/1238", "percentage": "96.69%", "elapsed_time": "14h 19m 31s", "remaining_time": "29m 26s"} +{"eps": 0, "objective/kl": 32.11440277, "objective/entropy": -101.44007111, "objective/non_score_reward": -1.60571992, "objective/rlhf_reward": 14.37865448, "objective/scores": 16.0, "policy/approxkl_avg": 0.00081715, "policy/clipfrac_avg": 0.00499542, "loss/policy_avg": -0.13895126, "loss/value_avg": 0.52845258, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15488815, "val/ratio": 0.98906481, "val/ratio_var": 7.767e-05, "val/num_eos_tokens": 443, "lr": 3e-08, "episode": 19168, "epoch": 0.96808081, "global_step/max_steps": "1198/1238", "percentage": "96.77%", "elapsed_time": "14h 19m 59s", "remaining_time": "28m 42s"} +{"eps": 0, "objective/kl": 37.39568329, "objective/entropy": -71.81754303, "objective/non_score_reward": -1.86978424, "objective/rlhf_reward": 13.52084064, "objective/scores": 15.375, "policy/approxkl_avg": 0.00087456, "policy/clipfrac_avg": 0.00250267, "loss/policy_avg": -0.22233413, "loss/value_avg": 0.69248414, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18534184, "val/ratio": 0.98760283, "val/ratio_var": 5.802e-05, "val/num_eos_tokens": 599, "lr": 3e-08, "episode": 19184, "epoch": 0.96888889, "global_step/max_steps": "1199/1238", "percentage": "96.85%", "elapsed_time": "14h 20m 47s", "remaining_time": "27m 59s"} +{"eps": 0, "objective/kl": 31.94279289, "objective/entropy": -74.75447083, "objective/non_score_reward": -1.5971396, "objective/rlhf_reward": 13.04348564, "objective/scores": 14.625, "policy/approxkl_avg": 0.00070554, "policy/clipfrac_avg": 0.00358203, "loss/policy_avg": 0.15578943, "loss/value_avg": 0.96131891, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1544745, "val/ratio": 0.99130297, "val/ratio_var": 5.023e-05, "val/num_eos_tokens": 539, "lr": 3e-08, "episode": 19200, "epoch": 0.96969697, "global_step/max_steps": "1200/1238", "percentage": "96.93%", "elapsed_time": "14h 21m 21s", "remaining_time": "27m 16s"} +{"eps": 0, "objective/kl": 37.90913391, "objective/entropy": -63.07292175, "objective/non_score_reward": -1.89545655, "objective/rlhf_reward": 14.11235619, "objective/scores": 16.0, "policy/approxkl_avg": 0.00083191, "policy/clipfrac_avg": 0.00485708, "loss/policy_avg": -0.19696119, "loss/value_avg": 0.46793222, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15787792, "val/ratio": 0.98986614, "val/ratio_var": 3.591e-05, "val/num_eos_tokens": 176, "lr": 3e-08, "episode": 19216, "epoch": 0.97050505, "global_step/max_steps": "1201/1238", "percentage": "97.01%", "elapsed_time": "14h 22m 15s", "remaining_time": "26m 33s"} +{"eps": 0, "objective/kl": 42.02083588, "objective/entropy": -85.22131348, "objective/non_score_reward": -2.10104179, "objective/rlhf_reward": 14.58645821, "objective/scores": 16.75, "policy/approxkl_avg": 0.00093885, "policy/clipfrac_avg": 0.00432074, "loss/policy_avg": -0.07627504, "loss/value_avg": 0.71020937, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18203044, "val/ratio": 0.98787516, "val/ratio_var": 8.285e-05, "val/num_eos_tokens": 108, "lr": 2e-08, "episode": 19232, "epoch": 0.97131313, "global_step/max_steps": "1202/1238", "percentage": "97.09%", "elapsed_time": "14h 22m 50s", "remaining_time": "25m 50s"} +{"eps": 0, "objective/kl": 33.10966492, "objective/entropy": -50.76046753, "objective/non_score_reward": -1.65548313, "objective/rlhf_reward": 14.21170425, "objective/scores": 15.875, "policy/approxkl_avg": 0.00103948, "policy/clipfrac_avg": 0.00573447, "loss/policy_avg": 0.02396649, "loss/value_avg": 0.3654055, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1836586, "val/ratio": 0.98744857, "val/ratio_var": 6.813e-05, "val/num_eos_tokens": 456, "lr": 2e-08, "episode": 19248, "epoch": 0.97212121, "global_step/max_steps": "1203/1238", "percentage": "97.17%", "elapsed_time": "14h 23m 23s", "remaining_time": "25m 7s"} +{"eps": 0, "objective/kl": 34.46487427, "objective/entropy": -41.68983078, "objective/non_score_reward": -1.72324371, "objective/rlhf_reward": 14.40956879, "objective/scores": 16.125, "policy/approxkl_avg": 0.00087818, "policy/clipfrac_avg": 0.00464723, "loss/policy_avg": 0.11596876, "loss/value_avg": 0.8759023, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18979645, "val/ratio": 0.98803526, "val/ratio_var": 7.466e-05, "val/num_eos_tokens": 8, "lr": 2e-08, "episode": 19264, "epoch": 0.97292929, "global_step/max_steps": "1204/1238", "percentage": "97.25%", "elapsed_time": "14h 24m 0s", "remaining_time": "24m 23s"} +{"eps": 0, "objective/kl": 38.24688339, "objective/entropy": -64.1900177, "objective/non_score_reward": -1.91234422, "objective/rlhf_reward": 12.51734352, "objective/scores": 14.4375, "policy/approxkl_avg": 0.000858, "policy/clipfrac_avg": 0.00243662, "loss/policy_avg": 0.0331149, "loss/value_avg": 0.37955981, "val/clipfrac_avg": 0.0065918, "policy/entropy_avg": 0.18629742, "val/ratio": 0.98809636, "val/ratio_var": 3.744e-05, "val/num_eos_tokens": 791, "lr": 2e-08, "episode": 19280, "epoch": 0.97373737, "global_step/max_steps": "1205/1238", "percentage": "97.33%", "elapsed_time": "14h 24m 44s", "remaining_time": "23m 40s"} +{"eps": 0, "objective/kl": 38.28486633, "objective/entropy": -47.87005615, "objective/non_score_reward": -1.91424334, "objective/rlhf_reward": 15.2966938, "objective/scores": 17.25, "policy/approxkl_avg": 0.00106832, "policy/clipfrac_avg": 0.00451328, "loss/policy_avg": -0.37829724, "loss/value_avg": 0.47165149, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21669468, "val/ratio": 0.9851523, "val/ratio_var": 8.786e-05, "val/num_eos_tokens": 24, "lr": 2e-08, "episode": 19296, "epoch": 0.97454545, "global_step/max_steps": "1206/1238", "percentage": "97.42%", "elapsed_time": "14h 25m 17s", "remaining_time": "22m 57s"} +{"eps": 0, "objective/kl": 45.99113464, "objective/entropy": -79.00741577, "objective/non_score_reward": -2.29955673, "objective/rlhf_reward": 13.70825577, "objective/scores": 16.0, "policy/approxkl_avg": 0.00092237, "policy/clipfrac_avg": 0.00405557, "loss/policy_avg": -0.12085377, "loss/value_avg": 0.53721482, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17890167, "val/ratio": 0.9878267, "val/ratio_var": 6.052e-05, "val/num_eos_tokens": 75, "lr": 2e-08, "episode": 19312, "epoch": 0.97535354, "global_step/max_steps": "1207/1238", "percentage": "97.50%", "elapsed_time": "14h 25m 59s", "remaining_time": "22m 14s"} +{"eps": 0, "objective/kl": 42.04967499, "objective/entropy": -66.35103607, "objective/non_score_reward": -2.10248399, "objective/rlhf_reward": 14.78032875, "objective/scores": 16.875, "policy/approxkl_avg": 0.00097626, "policy/clipfrac_avg": 0.00420854, "loss/policy_avg": -0.08923263, "loss/value_avg": 0.51029152, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19926167, "val/ratio": 0.98728824, "val/ratio_var": 6.779e-05, "val/num_eos_tokens": 505, "lr": 2e-08, "episode": 19328, "epoch": 0.97616162, "global_step/max_steps": "1208/1238", "percentage": "97.58%", "elapsed_time": "14h 26m 38s", "remaining_time": "21m 31s"} +{"eps": 0, "objective/kl": 42.12271881, "objective/entropy": -112.43741608, "objective/non_score_reward": -2.10613585, "objective/rlhf_reward": 11.19855118, "objective/scores": 13.3125, "policy/approxkl_avg": 0.00092499, "policy/clipfrac_avg": 0.00442465, "loss/policy_avg": 0.10433784, "loss/value_avg": 0.84282887, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19216204, "val/ratio": 0.98790514, "val/ratio_var": 6.321e-05, "val/num_eos_tokens": 27, "lr": 2e-08, "episode": 19344, "epoch": 0.9769697, "global_step/max_steps": "1209/1238", "percentage": "97.66%", "elapsed_time": "14h 27m 18s", "remaining_time": "20m 48s"} +{"eps": 0, "objective/kl": 34.45908356, "objective/entropy": -77.94785309, "objective/non_score_reward": -1.72295427, "objective/rlhf_reward": 14.1051712, "objective/scores": 15.8125, "policy/approxkl_avg": 0.00102094, "policy/clipfrac_avg": 0.00414194, "loss/policy_avg": 0.16060814, "loss/value_avg": 0.33783212, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18428802, "val/ratio": 0.98768955, "val/ratio_var": 0.0001052, "val/num_eos_tokens": 65, "lr": 1e-08, "episode": 19360, "epoch": 0.97777778, "global_step/max_steps": "1210/1238", "percentage": "97.74%", "elapsed_time": "14h 28m 3s", "remaining_time": "20m 5s"} +{"eps": 0, "objective/kl": 35.14643097, "objective/entropy": -46.33203125, "objective/non_score_reward": -1.75732172, "objective/rlhf_reward": 13.65674019, "objective/scores": 15.4375, "policy/approxkl_avg": 0.00097613, "policy/clipfrac_avg": 0.00375216, "loss/policy_avg": -0.06701843, "loss/value_avg": 0.36687267, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19565201, "val/ratio": 0.98752302, "val/ratio_var": 6.483e-05, "val/num_eos_tokens": 46, "lr": 1e-08, "episode": 19376, "epoch": 0.97858586, "global_step/max_steps": "1211/1238", "percentage": "97.82%", "elapsed_time": "14h 28m 29s", "remaining_time": "19m 21s"} +{"eps": 0, "objective/kl": 28.53516388, "objective/entropy": -69.58908844, "objective/non_score_reward": -1.42675805, "objective/rlhf_reward": 14.30761719, "objective/scores": 15.75, "policy/approxkl_avg": 0.00066858, "policy/clipfrac_avg": 0.00294222, "loss/policy_avg": -0.23902096, "loss/value_avg": 0.72444338, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16259003, "val/ratio": 0.9912442, "val/ratio_var": 4.313e-05, "val/num_eos_tokens": 143, "lr": 1e-08, "episode": 19392, "epoch": 0.97939394, "global_step/max_steps": "1212/1238", "percentage": "97.90%", "elapsed_time": "14h 29m 1s", "remaining_time": "18m 38s"} +{"eps": 0, "objective/kl": 41.61517715, "objective/entropy": -82.74155426, "objective/non_score_reward": -2.08075905, "objective/rlhf_reward": 14.60674095, "objective/scores": 16.75, "policy/approxkl_avg": 0.00094619, "policy/clipfrac_avg": 0.00698039, "loss/policy_avg": -0.11228986, "loss/value_avg": 0.53882003, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18620682, "val/ratio": 0.98767912, "val/ratio_var": 6.927e-05, "val/num_eos_tokens": 134, "lr": 1e-08, "episode": 19408, "epoch": 0.98020202, "global_step/max_steps": "1213/1238", "percentage": "97.98%", "elapsed_time": "14h 29m 30s", "remaining_time": "17m 55s"} +{"eps": 0, "objective/kl": 38.9472084, "objective/entropy": -104.67575073, "objective/non_score_reward": -1.9473604, "objective/rlhf_reward": 14.27138901, "objective/scores": 16.25, "policy/approxkl_avg": 0.00079256, "policy/clipfrac_avg": 0.00253249, "loss/policy_avg": -0.20061983, "loss/value_avg": 0.2869342, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15836048, "val/ratio": 0.98946178, "val/ratio_var": 4.717e-05, "val/num_eos_tokens": 256, "lr": 1e-08, "episode": 19424, "epoch": 0.9810101, "global_step/max_steps": "1214/1238", "percentage": "98.06%", "elapsed_time": "14h 30m 27s", "remaining_time": "17m 12s"} +{"eps": 0, "objective/kl": 41.90412521, "objective/entropy": -101.33221436, "objective/non_score_reward": -2.0952065, "objective/rlhf_reward": 14.52979279, "objective/scores": 16.625, "policy/approxkl_avg": 0.00087229, "policy/clipfrac_avg": 0.00251385, "loss/policy_avg": -0.12940247, "loss/value_avg": 0.50628233, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17874455, "val/ratio": 0.98772871, "val/ratio_var": 6.87e-05, "val/num_eos_tokens": 248, "lr": 1e-08, "episode": 19440, "epoch": 0.98181818, "global_step/max_steps": "1215/1238", "percentage": "98.14%", "elapsed_time": "14h 30m 59s", "remaining_time": "16m 29s"} +{"eps": 0, "objective/kl": 35.63471985, "objective/entropy": -192.84432983, "objective/non_score_reward": -1.78173614, "objective/rlhf_reward": 11.92138958, "objective/scores": 13.6875, "policy/approxkl_avg": 0.00094291, "policy/clipfrac_avg": 0.00262137, "loss/policy_avg": -0.39164668, "loss/value_avg": 1.55012739, "val/clipfrac_avg": 7.63e-06, "policy/entropy_avg": 0.17586327, "val/ratio": 0.98863119, "val/ratio_var": 7.117e-05, "val/num_eos_tokens": 61, "lr": 1e-08, "episode": 19456, "epoch": 0.98262626, "global_step/max_steps": "1216/1238", "percentage": "98.22%", "elapsed_time": "14h 32m 44s", "remaining_time": "15m 47s"} +{"eps": 0, "objective/kl": 39.29818726, "objective/entropy": -42.98954391, "objective/non_score_reward": -1.96490932, "objective/rlhf_reward": 14.4725914, "objective/scores": 16.5, "policy/approxkl_avg": 0.00095665, "policy/clipfrac_avg": 0.00446178, "loss/policy_avg": 0.08832584, "loss/value_avg": 0.43031436, "val/clipfrac_avg": 0.00195312, "policy/entropy_avg": 0.18021584, "val/ratio": 0.98645651, "val/ratio_var": 3.809e-05, "val/num_eos_tokens": 243, "lr": 1e-08, "episode": 19472, "epoch": 0.98343434, "global_step/max_steps": "1217/1238", "percentage": "98.30%", "elapsed_time": "14h 33m 15s", "remaining_time": "15m 4s"} +{"eps": 0, "objective/kl": 37.92512512, "objective/entropy": -107.35826111, "objective/non_score_reward": -1.89625645, "objective/rlhf_reward": 13.0803051, "objective/scores": 15.0, "policy/approxkl_avg": 0.00090071, "policy/clipfrac_avg": 0.0045885, "loss/policy_avg": 0.21013334, "loss/value_avg": 0.41512752, "val/clipfrac_avg": 0.00074405, "policy/entropy_avg": 0.18993211, "val/ratio": 0.98821044, "val/ratio_var": 7.512e-05, "val/num_eos_tokens": 164, "lr": 1e-08, "episode": 19488, "epoch": 0.98424242, "global_step/max_steps": "1218/1238", "percentage": "98.38%", "elapsed_time": "14h 34m 2s", "remaining_time": "14m 21s"} +{"eps": 0, "objective/kl": 42.53811646, "objective/entropy": -103.62552643, "objective/non_score_reward": -2.12690592, "objective/rlhf_reward": 13.98246861, "objective/scores": 16.125, "policy/approxkl_avg": 0.00081237, "policy/clipfrac_avg": 0.0042569, "loss/policy_avg": 0.16311818, "loss/value_avg": 0.31726879, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16496849, "val/ratio": 0.98979247, "val/ratio_var": 3.717e-05, "val/num_eos_tokens": 67, "lr": 1e-08, "episode": 19504, "epoch": 0.98505051, "global_step/max_steps": "1219/1238", "percentage": "98.47%", "elapsed_time": "14h 34m 42s", "remaining_time": "13m 38s"} +{"eps": 0, "objective/kl": 44.7612915, "objective/entropy": -120.69778442, "objective/non_score_reward": -2.23806453, "objective/rlhf_reward": 14.80099869, "objective/scores": 17.0, "policy/approxkl_avg": 0.00094881, "policy/clipfrac_avg": 0.00366858, "loss/policy_avg": -0.18715566, "loss/value_avg": 0.56154871, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1759553, "val/ratio": 0.9875381, "val/ratio_var": 7.448e-05, "val/num_eos_tokens": 949, "lr": 1e-08, "episode": 19520, "epoch": 0.98585859, "global_step/max_steps": "1220/1238", "percentage": "98.55%", "elapsed_time": "14h 35m 33s", "remaining_time": "12m 55s"} +{"eps": 0, "objective/kl": 42.00151062, "objective/entropy": -120.10693359, "objective/non_score_reward": -2.10007548, "objective/rlhf_reward": 14.21242428, "objective/scores": 16.25, "policy/approxkl_avg": 0.00087498, "policy/clipfrac_avg": 0.00498818, "loss/policy_avg": -0.42478049, "loss/value_avg": 0.9726249, "val/clipfrac_avg": 0.00023135, "policy/entropy_avg": 0.1530447, "val/ratio": 0.9896518, "val/ratio_var": 5.084e-05, "val/num_eos_tokens": 39, "lr": 1e-08, "episode": 19536, "epoch": 0.98666667, "global_step/max_steps": "1221/1238", "percentage": "98.63%", "elapsed_time": "14h 36m 21s", "remaining_time": "12m 12s"} +{"eps": 0, "objective/kl": 36.87531281, "objective/entropy": -129.04684448, "objective/non_score_reward": -1.84376574, "objective/rlhf_reward": 13.23045349, "objective/scores": 15.0625, "policy/approxkl_avg": 0.00076346, "policy/clipfrac_avg": 0.00551306, "loss/policy_avg": 0.17928611, "loss/value_avg": 0.83465528, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14986801, "val/ratio": 0.99037749, "val/ratio_var": 5.781e-05, "val/num_eos_tokens": 563, "lr": 1e-08, "episode": 19552, "epoch": 0.98747475, "global_step/max_steps": "1222/1238", "percentage": "98.71%", "elapsed_time": "14h 36m 58s", "remaining_time": "11m 28s"} +{"eps": 0, "objective/kl": 39.41420746, "objective/entropy": -49.08431625, "objective/non_score_reward": -1.97071052, "objective/rlhf_reward": 13.77147675, "objective/scores": 15.75, "policy/approxkl_avg": 0.00100286, "policy/clipfrac_avg": 0.00550021, "loss/policy_avg": -0.02808176, "loss/value_avg": 0.38955271, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20126534, "val/ratio": 0.98722428, "val/ratio_var": 4.987e-05, "val/num_eos_tokens": 38, "lr": 0.0, "episode": 19568, "epoch": 0.98828283, "global_step/max_steps": "1223/1238", "percentage": "98.79%", "elapsed_time": "14h 37m 23s", "remaining_time": "10m 45s"} +{"eps": 0, "objective/kl": 37.38958359, "objective/entropy": -40.71008301, "objective/non_score_reward": -1.86947918, "objective/rlhf_reward": 13.50552082, "objective/scores": 15.375, "policy/approxkl_avg": 0.00119487, "policy/clipfrac_avg": 0.0052036, "loss/policy_avg": -0.01493126, "loss/value_avg": 0.48372111, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.22698021, "val/ratio": 0.98559314, "val/ratio_var": 7.236e-05, "val/num_eos_tokens": 375, "lr": 0.0, "episode": 19584, "epoch": 0.98909091, "global_step/max_steps": "1224/1238", "percentage": "98.87%", "elapsed_time": "14h 37m 55s", "remaining_time": "10m 2s"} +{"eps": 0, "objective/kl": 40.45851898, "objective/entropy": -75.71792603, "objective/non_score_reward": -2.02292609, "objective/rlhf_reward": 14.32863617, "objective/scores": 16.375, "policy/approxkl_avg": 0.00097839, "policy/clipfrac_avg": 0.00296701, "loss/policy_avg": 0.05002763, "loss/value_avg": 0.33766478, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19454956, "val/ratio": 0.98637313, "val/ratio_var": 4.958e-05, "val/num_eos_tokens": 90, "lr": 0.0, "episode": 19600, "epoch": 0.98989899, "global_step/max_steps": "1225/1238", "percentage": "98.95%", "elapsed_time": "14h 38m 29s", "remaining_time": "9m 19s"} +{"eps": 0, "objective/kl": 45.66817474, "objective/entropy": -77.89369202, "objective/non_score_reward": -2.28340864, "objective/rlhf_reward": 14.65409088, "objective/scores": 17.0, "policy/approxkl_avg": 0.00086919, "policy/clipfrac_avg": 0.00276262, "loss/policy_avg": -0.09883705, "loss/value_avg": 0.40289462, "val/clipfrac_avg": 0.00028617, "policy/entropy_avg": 0.18253517, "val/ratio": 0.98847616, "val/ratio_var": 5.125e-05, "val/num_eos_tokens": 48, "lr": 0.0, "episode": 19616, "epoch": 0.99070707, "global_step/max_steps": "1226/1238", "percentage": "99.03%", "elapsed_time": "14h 39m 12s", "remaining_time": "8m 36s"} +{"eps": 0, "objective/kl": 41.73491669, "objective/entropy": -133.75288391, "objective/non_score_reward": -2.08674574, "objective/rlhf_reward": 13.96012878, "objective/scores": 16.0, "policy/approxkl_avg": 0.00072043, "policy/clipfrac_avg": 0.0029739, "loss/policy_avg": 0.02128005, "loss/value_avg": 0.34802741, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14430571, "val/ratio": 0.99030411, "val/ratio_var": 6.999e-05, "val/num_eos_tokens": 316, "lr": 0.0, "episode": 19632, "epoch": 0.99151515, "global_step/max_steps": "1227/1238", "percentage": "99.11%", "elapsed_time": "14h 39m 50s", "remaining_time": "7m 53s"} +{"eps": 0, "objective/kl": 42.03723526, "objective/entropy": -44.44089127, "objective/non_score_reward": -2.10186195, "objective/rlhf_reward": 14.17938805, "objective/scores": 16.25, "policy/approxkl_avg": 0.00091578, "policy/clipfrac_avg": 0.00324251, "loss/policy_avg": -0.30528533, "loss/value_avg": 0.75302458, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19690704, "val/ratio": 0.9876858, "val/ratio_var": 5.984e-05, "val/num_eos_tokens": 429, "lr": 0.0, "episode": 19648, "epoch": 0.99232323, "global_step/max_steps": "1228/1238", "percentage": "99.19%", "elapsed_time": "14h 40m 32s", "remaining_time": "7m 10s"} +{"eps": 0, "objective/kl": 43.74427414, "objective/entropy": -50.87998962, "objective/non_score_reward": -2.1872139, "objective/rlhf_reward": 15.8127861, "objective/scores": 18.0, "policy/approxkl_avg": 0.00096512, "policy/clipfrac_avg": 0.00359187, "loss/policy_avg": 0.07529311, "loss/value_avg": 0.36114734, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19472504, "val/ratio": 0.98762703, "val/ratio_var": 4.564e-05, "val/num_eos_tokens": 331, "lr": 0.0, "episode": 19664, "epoch": 0.99313131, "global_step/max_steps": "1229/1238", "percentage": "99.27%", "elapsed_time": "14h 41m 16s", "remaining_time": "6m 27s"} +{"eps": 0, "objective/kl": 38.0479126, "objective/entropy": -69.65153503, "objective/non_score_reward": -1.90239584, "objective/rlhf_reward": 13.4257288, "objective/scores": 15.3125, "policy/approxkl_avg": 0.00083254, "policy/clipfrac_avg": 0.00312162, "loss/policy_avg": -0.08426039, "loss/value_avg": 0.34303862, "val/clipfrac_avg": 0.00107759, "policy/entropy_avg": 0.17328572, "val/ratio": 0.98911262, "val/ratio_var": 5.815e-05, "val/num_eos_tokens": 327, "lr": 0.0, "episode": 19680, "epoch": 0.99393939, "global_step/max_steps": "1230/1238", "percentage": "99.35%", "elapsed_time": "14h 41m 53s", "remaining_time": "5m 44s"} +{"eps": 0, "objective/kl": 36.74051666, "objective/entropy": -27.6818676, "objective/non_score_reward": -1.83702588, "objective/rlhf_reward": 13.7410984, "objective/scores": 15.5625, "policy/approxkl_avg": 0.00090048, "policy/clipfrac_avg": 0.00513545, "loss/policy_avg": 0.00234514, "loss/value_avg": 1.12141919, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18401337, "val/ratio": 0.98841548, "val/ratio_var": 7.396e-05, "val/num_eos_tokens": 137, "lr": 0.0, "episode": 19696, "epoch": 0.99474747, "global_step/max_steps": "1231/1238", "percentage": "99.43%", "elapsed_time": "14h 42m 39s", "remaining_time": "5m 1s"} +{"eps": 0, "objective/kl": 45.0079689, "objective/entropy": -45.30944061, "objective/non_score_reward": -2.25039864, "objective/rlhf_reward": 15.55428886, "objective/scores": 17.75, "policy/approxkl_avg": 0.00096042, "policy/clipfrac_avg": 0.00451303, "loss/policy_avg": -0.16311845, "loss/value_avg": 0.45753658, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18356657, "val/ratio": 0.98890924, "val/ratio_var": 4.462e-05, "val/num_eos_tokens": 4, "lr": 0.0, "episode": 19712, "epoch": 0.99555556, "global_step/max_steps": "1232/1238", "percentage": "99.52%", "elapsed_time": "14h 43m 29s", "remaining_time": "4m 18s"} +{"eps": 0, "objective/kl": 37.37506104, "objective/entropy": -19.21195221, "objective/non_score_reward": -1.86875308, "objective/rlhf_reward": 12.95937157, "objective/scores": 14.8125, "policy/approxkl_avg": 0.00089314, "policy/clipfrac_avg": 0.00383983, "loss/policy_avg": -0.26251727, "loss/value_avg": 0.80983853, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17013741, "val/ratio": 0.98914987, "val/ratio_var": 7.125e-05, "val/num_eos_tokens": 12, "lr": 0.0, "episode": 19728, "epoch": 0.99636364, "global_step/max_steps": "1233/1238", "percentage": "99.60%", "elapsed_time": "14h 44m 3s", "remaining_time": "3m 35s"} +{"eps": 0, "objective/kl": 46.30189896, "objective/entropy": -53.28070831, "objective/non_score_reward": -2.31509495, "objective/rlhf_reward": 14.2630291, "objective/scores": 16.625, "policy/approxkl_avg": 0.00100941, "policy/clipfrac_avg": 0.00642228, "loss/policy_avg": 0.10235727, "loss/value_avg": 0.41452596, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19713116, "val/ratio": 0.98699343, "val/ratio_var": 9.036e-05, "val/num_eos_tokens": 349, "lr": 0.0, "episode": 19744, "epoch": 0.99717172, "global_step/max_steps": "1234/1238", "percentage": "99.68%", "elapsed_time": "14h 44m 34s", "remaining_time": "2m 52s"} +{"eps": 0, "objective/kl": 37.48680878, "objective/entropy": -68.47766113, "objective/non_score_reward": -1.87434053, "objective/rlhf_reward": 14.49284744, "objective/scores": 16.375, "policy/approxkl_avg": 0.00094368, "policy/clipfrac_avg": 0.00542159, "loss/policy_avg": -0.13045719, "loss/value_avg": 0.82330263, "val/clipfrac_avg": 4.183e-05, "policy/entropy_avg": 0.17751884, "val/ratio": 0.98851186, "val/ratio_var": 5.037e-05, "val/num_eos_tokens": 52, "lr": 0.0, "episode": 19760, "epoch": 0.9979798, "global_step/max_steps": "1235/1238", "percentage": "99.76%", "elapsed_time": "14h 45m 12s", "remaining_time": "2m 9s"} +{"eps": 0, "objective/kl": 44.79389191, "objective/entropy": -155.75683594, "objective/non_score_reward": -2.2396946, "objective/rlhf_reward": 14.1196804, "objective/scores": 16.375, "policy/approxkl_avg": 0.0007429, "policy/clipfrac_avg": 0.00298373, "loss/policy_avg": -0.2191502, "loss/value_avg": 0.6580562, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14424515, "val/ratio": 0.99012548, "val/ratio_var": 6.484e-05, "val/num_eos_tokens": 405, "lr": 0.0, "episode": 19776, "epoch": 0.99878788, "global_step/max_steps": "1236/1238", "percentage": "99.84%", "elapsed_time": "14h 45m 54s", "remaining_time": "1m 26s"} +{"eps": 0, "objective/kl": 38.98764038, "objective/entropy": -55.40583801, "objective/non_score_reward": -1.94938231, "objective/rlhf_reward": 12.29280567, "objective/scores": 14.25, "policy/approxkl_avg": 0.00082455, "policy/clipfrac_avg": 0.00260689, "loss/policy_avg": 0.1954039, "loss/value_avg": 1.04959536, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16810131, "val/ratio": 0.98874998, "val/ratio_var": 4.212e-05, "val/num_eos_tokens": 134, "lr": 0.0, "episode": 19792, "epoch": 0.99959596, "global_step/max_steps": "1237/1238", "percentage": "99.92%", "elapsed_time": "14h 46m 25s", "remaining_time": "42s"} +{"eps": 0, "objective/kl": 39.11707306, "objective/entropy": -112.83091736, "objective/non_score_reward": -1.9558537, "objective/rlhf_reward": 11.75898933, "objective/scores": 13.6875, "policy/approxkl_avg": 0.00083753, "policy/clipfrac_avg": 0.00415769, "loss/policy_avg": 0.02539672, "loss/value_avg": 0.54931873, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15214157, "val/ratio": 0.99000144, "val/ratio_var": 6.578e-05, "val/num_eos_tokens": 299, "lr": 0.0, "episode": 19808, "epoch": 1.00040404, "global_step/max_steps": "1238/1238", "percentage": "100.00%", "elapsed_time": "14h 47m 26s", "remaining_time": "0s"} +{"value_model_parameter_info": "PeftModelForSequenceClassification: 7525.9085M Params (20.9756M Trainable [0.2787%]), 0.0001M Buffers.", "train_dataset": "52.356818±12.681108, min=39.000000, max=284.000000, size=19800", "val_dataset": "53.240000±11.806456, min=40.000000, max=111.000000, size=200", "model_parameter_info": "PeftModelForCausalLM: 8051.2328M Params (20.9715M Trainable [0.2605%]), 0.0001M Buffers.", "last_model_checkpoint": "/DIR-reproduction/outputs/ppo_dir_v2/v0-20260524-184726/checkpoint-1238", "best_model_checkpoint": null, "best_metric": null, "global_step": 1238, "log_history": [{"eps": 0, "objective/kl": 4.434216499328613, "objective/entropy": -74.55732727050781, "objective/non_score_reward": -0.22171086072921753, "objective/rlhf_reward": 9.188445091247559, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0009155935840681195, "policy/clipfrac_avg": 0.0011241745669394732, "loss/policy_avg": -0.2878437042236328, "loss/value_avg": 10.6063871383667, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19521331787109375, "val/ratio": 0.9892521500587463, "val/ratio_var": 5.706197043764405e-05, "val/num_eos_tokens": 545, "lr": 0.0, "episode": 16, "epoch": 0.0008080808080808081, "step": 1}, {"eps": 0, "objective/kl": 8.247895240783691, "objective/entropy": -73.15070343017578, "objective/non_score_reward": -0.41239476203918457, "objective/rlhf_reward": 8.626667976379395, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0011615954572334886, "policy/clipfrac_avg": 0.0034806979820132256, "loss/policy_avg": -0.23873865604400635, "loss/value_avg": 8.199307441711426, "val/clipfrac_avg": 0.00031833729008212686, "policy/entropy_avg": 0.26854443550109863, "val/ratio": 0.9843853712081909, "val/ratio_var": 0.00014292627747636288, "val/num_eos_tokens": 16, "lr": 1.6129032258064518e-07, "episode": 32, "epoch": 0.0016161616161616162, "step": 2}, {"eps": 0, "objective/kl": 5.790386199951172, "objective/entropy": -57.60097885131836, "objective/non_score_reward": -0.2895193099975586, "objective/rlhf_reward": 7.066926002502441, "objective/scores": 7.34375, "policy/approxkl_avg": 0.0009981861803680658, "policy/clipfrac_avg": 0.003842724720016122, "loss/policy_avg": -0.17912136018276215, "loss/value_avg": 9.051790237426758, "val/clipfrac_avg": 0.00032501743407920003, "policy/entropy_avg": 0.2743988037109375, "val/ratio": 0.9859670400619507, "val/ratio_var": 7.310912042157724e-05, "val/num_eos_tokens": 85, "lr": 3.2258064516129035e-07, "episode": 48, "epoch": 0.0024242424242424242, "step": 3}, {"eps": 0, "objective/kl": 8.576089859008789, "objective/entropy": -63.22553634643555, "objective/non_score_reward": -0.42880451679229736, "objective/rlhf_reward": 7.743070125579834, "objective/scores": 8.1875, "policy/approxkl_avg": 0.001054076012223959, "policy/clipfrac_avg": 0.0036023426800966263, "loss/policy_avg": -0.11512397229671478, "loss/value_avg": 6.804945945739746, "val/clipfrac_avg": 0.0008609364158473909, "policy/entropy_avg": 0.2464141845703125, "val/ratio": 0.9846372604370117, "val/ratio_var": 7.18827432137914e-05, "val/num_eos_tokens": 422, "lr": 4.838709677419355e-07, "episode": 64, "epoch": 0.0032323232323232323, "step": 4}, {"eps": 0, "objective/kl": 5.362919807434082, "objective/entropy": -114.28401184082031, "objective/non_score_reward": -0.26814600825309753, "objective/rlhf_reward": 7.38810396194458, "objective/scores": 7.65625, "policy/approxkl_avg": 0.0007407054654322565, "policy/clipfrac_avg": 0.004495600238442421, "loss/policy_avg": -0.07829581201076508, "loss/value_avg": 6.955084323883057, "val/clipfrac_avg": 0.0008702859631739557, "policy/entropy_avg": 0.16756057739257812, "val/ratio": 0.9889297485351562, "val/ratio_var": 4.611695112544112e-05, "val/num_eos_tokens": 962, "lr": 6.451612903225807e-07, "episode": 80, "epoch": 0.00404040404040404, "step": 5}, {"eps": 0, "objective/kl": 5.353013038635254, "objective/entropy": -120.7701644897461, "objective/non_score_reward": -0.26765066385269165, "objective/rlhf_reward": 8.775318145751953, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0007811508257873356, "policy/clipfrac_avg": 0.0025352579541504383, "loss/policy_avg": -0.2765856385231018, "loss/value_avg": 9.976322174072266, "val/clipfrac_avg": 0.0010816564317792654, "policy/entropy_avg": 0.21323394775390625, "val/ratio": 0.988487720489502, "val/ratio_var": 6.569296238012612e-05, "val/num_eos_tokens": 128, "lr": 8.064516129032258e-07, "episode": 96, "epoch": 0.0048484848484848485, "step": 6}, {"eps": 0, "objective/kl": 6.929623126983643, "objective/entropy": -57.45833969116211, "objective/non_score_reward": -0.3464811444282532, "objective/rlhf_reward": 7.094925403594971, "objective/scores": 7.4375, "policy/approxkl_avg": 0.0009421511786058545, "policy/clipfrac_avg": 0.00409944262355566, "loss/policy_avg": -0.036815427243709564, "loss/value_avg": 5.047958850860596, "val/clipfrac_avg": 0.0008602802408859134, "policy/entropy_avg": 0.206146240234375, "val/ratio": 0.9869779348373413, "val/ratio_var": 3.1514064176008105e-05, "val/num_eos_tokens": 150, "lr": 9.67741935483871e-07, "episode": 112, "epoch": 0.0056565656565656566, "step": 7}, {"eps": 0, "objective/kl": 4.520520210266113, "objective/entropy": -4.442367076873779, "objective/non_score_reward": -0.22602598369121552, "objective/rlhf_reward": 11.03569221496582, "objective/scores": 11.25, "policy/approxkl_avg": 0.0011083385907113552, "policy/clipfrac_avg": 0.004277850966900587, "loss/policy_avg": -0.13899217545986176, "loss/value_avg": 12.854679107666016, "val/clipfrac_avg": 0.001243304694071412, "policy/entropy_avg": 0.23640060424804688, "val/ratio": 0.9858227968215942, "val/ratio_var": 6.114065763540566e-05, "val/num_eos_tokens": 4, "lr": 1.1290322580645162e-06, "episode": 128, "epoch": 0.006464646464646465, "step": 8}, {"eps": 0, "objective/kl": 4.24302339553833, "objective/entropy": -90.28558349609375, "objective/non_score_reward": -0.2121511697769165, "objective/rlhf_reward": 8.588630676269531, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0007802502950653434, "policy/clipfrac_avg": 0.0023238342255353928, "loss/policy_avg": -0.19064569473266602, "loss/value_avg": 16.10247802734375, "val/clipfrac_avg": 0.0005082917050458491, "policy/entropy_avg": 0.16461068391799927, "val/ratio": 0.9889500737190247, "val/ratio_var": 5.1900708058383316e-05, "val/num_eos_tokens": 11, "lr": 1.2903225806451614e-06, "episode": 144, "epoch": 0.007272727272727273, "step": 9}, {"eps": 0, "objective/kl": 8.206380844116211, "objective/entropy": -120.9984359741211, "objective/non_score_reward": -0.41031908988952637, "objective/rlhf_reward": 6.3904619216918945, "objective/scores": 6.8125, "policy/approxkl_avg": 0.0009577925666235387, "policy/clipfrac_avg": 0.005130666308104992, "loss/policy_avg": -0.13336265087127686, "loss/value_avg": 5.764339923858643, "val/clipfrac_avg": 0.0019272974459454417, "policy/entropy_avg": 0.23892974853515625, "val/ratio": 0.9874762296676636, "val/ratio_var": 6.36352488072589e-05, "val/num_eos_tokens": 368, "lr": 1.4516129032258066e-06, "episode": 160, "epoch": 0.00808080808080808, "step": 10}, {"eps": 0, "objective/kl": 5.863861560821533, "objective/entropy": -47.950660705566406, "objective/non_score_reward": -0.29319310188293457, "objective/rlhf_reward": 8.073994636535645, "objective/scores": 8.375, "policy/approxkl_avg": 0.0011588437482714653, "policy/clipfrac_avg": 0.0038244768511503935, "loss/policy_avg": -0.1956510990858078, "loss/value_avg": 9.056394577026367, "val/clipfrac_avg": 0.0022193677723407745, "policy/entropy_avg": 0.2566804885864258, "val/ratio": 0.9839282035827637, "val/ratio_var": 9.391677303938195e-05, "val/num_eos_tokens": 395, "lr": 1.6129032258064516e-06, "episode": 176, "epoch": 0.008888888888888889, "step": 11}, {"eps": 0, "objective/kl": 7.083408832550049, "objective/entropy": -56.2935791015625, "objective/non_score_reward": -0.35417044162750244, "objective/rlhf_reward": 8.083329200744629, "objective/scores": 8.4375, "policy/approxkl_avg": 0.001030836603604257, "policy/clipfrac_avg": 0.004677055403590202, "loss/policy_avg": -0.09844052791595459, "loss/value_avg": 5.998026371002197, "val/clipfrac_avg": 0.0010991215240210295, "policy/entropy_avg": 0.21318721771240234, "val/ratio": 0.9859715700149536, "val/ratio_var": 4.26560545747634e-05, "val/num_eos_tokens": 36, "lr": 1.774193548387097e-06, "episode": 192, "epoch": 0.009696969696969697, "step": 12}, {"eps": 0, "objective/kl": 6.387138843536377, "objective/entropy": -47.072654724121094, "objective/non_score_reward": -0.31935691833496094, "objective/rlhf_reward": 6.657205581665039, "objective/scores": 6.96875, "policy/approxkl_avg": 0.0011611657682806253, "policy/clipfrac_avg": 0.005622599273920059, "loss/policy_avg": -0.10422849655151367, "loss/value_avg": 7.215424060821533, "val/clipfrac_avg": 0.005995843559503555, "policy/entropy_avg": 0.25177001953125, "val/ratio": 0.9847086071968079, "val/ratio_var": 6.94288028171286e-05, "val/num_eos_tokens": 541, "lr": 1.935483870967742e-06, "episode": 208, "epoch": 0.010505050505050505, "step": 13}, {"eps": 0, "objective/kl": 6.544907569885254, "objective/entropy": -41.007904052734375, "objective/non_score_reward": -0.3272453844547272, "objective/rlhf_reward": 8.280176162719727, "objective/scores": 8.625, "policy/approxkl_avg": 0.001087480690330267, "policy/clipfrac_avg": 0.00600704550743103, "loss/policy_avg": -0.12459716200828552, "loss/value_avg": 7.5816330909729, "val/clipfrac_avg": 0.005673446226865053, "policy/entropy_avg": 0.26541900634765625, "val/ratio": 0.9857578277587891, "val/ratio_var": 9.339387906948105e-05, "val/num_eos_tokens": 407, "lr": 2.096774193548387e-06, "episode": 224, "epoch": 0.011313131313131313, "step": 14}, {"eps": 0, "objective/kl": 6.37682580947876, "objective/entropy": -152.62283325195312, "objective/non_score_reward": -0.31884127855300903, "objective/rlhf_reward": 8.247565269470215, "objective/scores": 8.5625, "policy/approxkl_avg": 0.0008043860434554517, "policy/clipfrac_avg": 0.004083259496837854, "loss/policy_avg": -0.3509424924850464, "loss/value_avg": 8.651493072509766, "val/clipfrac_avg": 0.02940819412469864, "policy/entropy_avg": 0.19263720512390137, "val/ratio": 0.9888343214988708, "val/ratio_var": 6.805222801631317e-05, "val/num_eos_tokens": 479, "lr": 2.2580645161290324e-06, "episode": 240, "epoch": 0.012121212121212121, "step": 15}, {"eps": 0, "objective/kl": 7.03504753112793, "objective/entropy": -69.35281372070312, "objective/non_score_reward": -0.3517524003982544, "objective/rlhf_reward": 8.050591468811035, "objective/scores": 8.375, "policy/approxkl_avg": 0.0011161768343299627, "policy/clipfrac_avg": 0.005170322954654694, "loss/policy_avg": -0.20472919940948486, "loss/value_avg": 7.683527946472168, "val/clipfrac_avg": 0.051273077726364136, "policy/entropy_avg": 0.2497091293334961, "val/ratio": 0.9848048686981201, "val/ratio_var": 8.986083412310109e-05, "val/num_eos_tokens": 17, "lr": 2.4193548387096776e-06, "episode": 256, "epoch": 0.01292929292929293, "step": 16}, {"eps": 0, "objective/kl": 6.518709659576416, "objective/entropy": -107.79791259765625, "objective/non_score_reward": -0.3259354829788208, "objective/rlhf_reward": 8.931877136230469, "objective/scores": 9.25, "policy/approxkl_avg": 0.0009423322044312954, "policy/clipfrac_avg": 0.0035665533505380154, "loss/policy_avg": -0.32048213481903076, "loss/value_avg": 13.413436889648438, "val/clipfrac_avg": 0.066550612449646, "policy/entropy_avg": 0.19956940412521362, "val/ratio": 0.9872732162475586, "val/ratio_var": 6.39159043203108e-05, "val/num_eos_tokens": 636, "lr": 2.580645161290323e-06, "episode": 272, "epoch": 0.013737373737373737, "step": 17}, {"eps": 0, "objective/kl": 3.65440034866333, "objective/entropy": -60.37420654296875, "objective/non_score_reward": -0.18272003531455994, "objective/rlhf_reward": 7.470600605010986, "objective/scores": 7.65625, "policy/approxkl_avg": 0.0009109527454711497, "policy/clipfrac_avg": 0.0032424808014184237, "loss/policy_avg": -0.16326521337032318, "loss/value_avg": 7.0222063064575195, "val/clipfrac_avg": 0.15316107869148254, "policy/entropy_avg": 0.21929168701171875, "val/ratio": 0.9875004291534424, "val/ratio_var": 6.943759944988415e-05, "val/num_eos_tokens": 376, "lr": 2.7419354838709676e-06, "episode": 288, "epoch": 0.014545454545454545, "step": 18}, {"eps": 0, "objective/kl": 8.19035530090332, "objective/entropy": -64.48834228515625, "objective/non_score_reward": -0.40951770544052124, "objective/rlhf_reward": 7.082670211791992, "objective/scores": 7.5, "policy/approxkl_avg": 0.0009629215346649289, "policy/clipfrac_avg": 0.004428057000041008, "loss/policy_avg": -0.09499570727348328, "loss/value_avg": 4.847567558288574, "val/clipfrac_avg": 0.09573161602020264, "policy/entropy_avg": 0.21021270751953125, "val/ratio": 0.9871885776519775, "val/ratio_var": 5.789758506580256e-05, "val/num_eos_tokens": 161, "lr": 2.903225806451613e-06, "episode": 304, "epoch": 0.015353535353535354, "step": 19}, {"eps": 0, "objective/kl": 4.762131690979004, "objective/entropy": -92.24372863769531, "objective/non_score_reward": -0.23810657858848572, "objective/rlhf_reward": 8.150565147399902, "objective/scores": 8.375, "policy/approxkl_avg": 0.0009718802757561207, "policy/clipfrac_avg": 0.004476544447243214, "loss/policy_avg": -0.36197370290756226, "loss/value_avg": 9.9912109375, "val/clipfrac_avg": 0.15394406020641327, "policy/entropy_avg": 0.261799693107605, "val/ratio": 0.9875181317329407, "val/ratio_var": 9.373524517286569e-05, "val/num_eos_tokens": 306, "lr": 3.0645161290322584e-06, "episode": 320, "epoch": 0.01616161616161616, "step": 20}, {"eps": 0, "objective/kl": 5.371654510498047, "objective/entropy": -60.66798400878906, "objective/non_score_reward": -0.26858270168304443, "objective/rlhf_reward": 6.957979679107666, "objective/scores": 7.21875, "policy/approxkl_avg": 0.0009720490779727697, "policy/clipfrac_avg": 0.002186079975217581, "loss/policy_avg": -0.06269250065088272, "loss/value_avg": 5.039106369018555, "val/clipfrac_avg": 0.11872587352991104, "policy/entropy_avg": 0.2316131591796875, "val/ratio": 0.9866783618927002, "val/ratio_var": 5.079948095954023e-05, "val/num_eos_tokens": 62, "lr": 3.225806451612903e-06, "episode": 336, "epoch": 0.01696969696969697, "step": 21}, {"eps": 0, "objective/kl": 3.4860763549804688, "objective/entropy": -52.22957229614258, "objective/non_score_reward": -0.1743038147687912, "objective/rlhf_reward": 8.28663444519043, "objective/scores": 8.4375, "policy/approxkl_avg": 0.0007850549882277846, "policy/clipfrac_avg": 0.004184068646281958, "loss/policy_avg": -0.1775662899017334, "loss/value_avg": 9.879995346069336, "val/clipfrac_avg": 0.19386646151542664, "policy/entropy_avg": 0.17317485809326172, "val/ratio": 0.9883713722229004, "val/ratio_var": 5.718659667763859e-05, "val/num_eos_tokens": 227, "lr": 3.3870967741935484e-06, "episode": 352, "epoch": 0.017777777777777778, "step": 22}, {"eps": 0, "objective/kl": 6.329047203063965, "objective/entropy": -17.842317581176758, "objective/non_score_reward": -0.31645238399505615, "objective/rlhf_reward": 9.144485473632812, "objective/scores": 9.4375, "policy/approxkl_avg": 0.001143642351962626, "policy/clipfrac_avg": 0.0046859681606292725, "loss/policy_avg": -0.08608929812908173, "loss/value_avg": 5.026681900024414, "val/clipfrac_avg": 0.11751493066549301, "policy/entropy_avg": 0.245330810546875, "val/ratio": 0.9848726987838745, "val/ratio_var": 6.24043750576675e-05, "val/num_eos_tokens": 412, "lr": 3.548387096774194e-06, "episode": 368, "epoch": 0.018585858585858588, "step": 23}, {"eps": 0, "objective/kl": 5.16103458404541, "objective/entropy": -47.785430908203125, "objective/non_score_reward": -0.2580517530441284, "objective/rlhf_reward": 7.452885627746582, "objective/scores": 7.71875, "policy/approxkl_avg": 0.0009465434122830629, "policy/clipfrac_avg": 0.003977151587605476, "loss/policy_avg": -0.010797667317092419, "loss/value_avg": 6.275961875915527, "val/clipfrac_avg": 0.12968723475933075, "policy/entropy_avg": 0.28936767578125, "val/ratio": 0.9866119027137756, "val/ratio_var": 6.631347787333652e-05, "val/num_eos_tokens": 200, "lr": 3.7096774193548392e-06, "episode": 384, "epoch": 0.019393939393939394, "step": 24}, {"eps": 0, "objective/kl": 7.756465911865234, "objective/entropy": -20.867111206054688, "objective/non_score_reward": -0.38782334327697754, "objective/rlhf_reward": 7.920770645141602, "objective/scores": 8.3125, "policy/approxkl_avg": 0.0012229372514411807, "policy/clipfrac_avg": 0.0030088755302131176, "loss/policy_avg": 0.03891272097826004, "loss/value_avg": 5.366772651672363, "val/clipfrac_avg": 0.12619256973266602, "policy/entropy_avg": 0.2554168701171875, "val/ratio": 0.9858287572860718, "val/ratio_var": 6.279556691879407e-05, "val/num_eos_tokens": 16, "lr": 3.870967741935484e-06, "episode": 400, "epoch": 0.020202020202020204, "step": 25}, {"eps": 0, "objective/kl": 6.121448040008545, "objective/entropy": -131.1887969970703, "objective/non_score_reward": -0.3060724139213562, "objective/rlhf_reward": 7.201740264892578, "objective/scores": 7.5, "policy/approxkl_avg": 0.0008630367228761315, "policy/clipfrac_avg": 0.003512686351314187, "loss/policy_avg": -0.09346847981214523, "loss/value_avg": 3.7916016578674316, "val/clipfrac_avg": 0.10546709597110748, "policy/entropy_avg": 0.2108306884765625, "val/ratio": 0.9874800443649292, "val/ratio_var": 6.414195377146825e-05, "val/num_eos_tokens": 212, "lr": 4.032258064516129e-06, "episode": 416, "epoch": 0.02101010101010101, "step": 26}, {"eps": 0, "objective/kl": 6.764763355255127, "objective/entropy": -91.82192993164062, "objective/non_score_reward": -0.3382381796836853, "objective/rlhf_reward": 9.751605987548828, "objective/scores": 10.0625, "policy/approxkl_avg": 0.000947782420553267, "policy/clipfrac_avg": 0.005451520439237356, "loss/policy_avg": -0.08439041674137115, "loss/value_avg": 4.865840435028076, "val/clipfrac_avg": 0.06509673595428467, "policy/entropy_avg": 0.209228515625, "val/ratio": 0.9870284795761108, "val/ratio_var": 4.723674283013679e-05, "val/num_eos_tokens": 221, "lr": 4.193548387096774e-06, "episode": 432, "epoch": 0.02181818181818182, "step": 27}, {"eps": 0, "objective/kl": 7.102960586547852, "objective/entropy": -68.857421875, "objective/non_score_reward": -0.3551480770111084, "objective/rlhf_reward": 8.937820434570312, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0010395422577857971, "policy/clipfrac_avg": 0.005196031183004379, "loss/policy_avg": -0.12459178268909454, "loss/value_avg": 4.973685264587402, "val/clipfrac_avg": 0.09726891666650772, "policy/entropy_avg": 0.22506141662597656, "val/ratio": 0.9859325885772705, "val/ratio_var": 7.372920663328841e-05, "val/num_eos_tokens": 153, "lr": 4.35483870967742e-06, "episode": 448, "epoch": 0.022626262626262626, "step": 28}, {"eps": 0, "objective/kl": 4.585806846618652, "objective/entropy": -39.28635787963867, "objective/non_score_reward": -0.22929036617279053, "objective/rlhf_reward": 7.24336576461792, "objective/scores": 7.46875, "policy/approxkl_avg": 0.0010784105397760868, "policy/clipfrac_avg": 0.004656917415559292, "loss/policy_avg": -0.0893494039773941, "loss/value_avg": 4.280435085296631, "val/clipfrac_avg": 0.033933281898498535, "policy/entropy_avg": 0.28299713134765625, "val/ratio": 0.9854025840759277, "val/ratio_var": 7.980425289133564e-05, "val/num_eos_tokens": 151, "lr": 4.516129032258065e-06, "episode": 464, "epoch": 0.023434343434343436, "step": 29}, {"eps": 0, "objective/kl": 3.2583465576171875, "objective/entropy": -56.16079330444336, "objective/non_score_reward": -0.1629173308610916, "objective/rlhf_reward": 6.471848487854004, "objective/scores": 6.625, "policy/approxkl_avg": 0.0009038803982548416, "policy/clipfrac_avg": 0.0039865439757704735, "loss/policy_avg": 0.0009674776811152697, "loss/value_avg": 3.2916879653930664, "val/clipfrac_avg": 0.037767596542835236, "policy/entropy_avg": 0.21944808959960938, "val/ratio": 0.9880707263946533, "val/ratio_var": 7.166655996115878e-05, "val/num_eos_tokens": 229, "lr": 4.67741935483871e-06, "episode": 480, "epoch": 0.024242424242424242, "step": 30}, {"eps": 0, "objective/kl": 4.62398624420166, "objective/entropy": -85.58920288085938, "objective/non_score_reward": -0.23119933903217316, "objective/rlhf_reward": 8.104738235473633, "objective/scores": 8.3125, "policy/approxkl_avg": 0.0008234091801568866, "policy/clipfrac_avg": 0.0037510073743760586, "loss/policy_avg": -0.05671697109937668, "loss/value_avg": 4.5554399490356445, "val/clipfrac_avg": 0.12869636714458466, "policy/entropy_avg": 0.18961238861083984, "val/ratio": 0.9884214997291565, "val/ratio_var": 7.101450319169089e-05, "val/num_eos_tokens": 30, "lr": 4.838709677419355e-06, "episode": 496, "epoch": 0.025050505050505052, "step": 31}, {"eps": 0, "objective/kl": 5.964879035949707, "objective/entropy": -81.65757751464844, "objective/non_score_reward": -0.2982439696788788, "objective/rlhf_reward": 6.811131000518799, "objective/scores": 7.125, "policy/approxkl_avg": 0.0008165108738467097, "policy/clipfrac_avg": 0.0019289799965918064, "loss/policy_avg": -0.08824437111616135, "loss/value_avg": 3.4218335151672363, "val/clipfrac_avg": 0.10708418488502502, "policy/entropy_avg": 0.1952981948852539, "val/ratio": 0.9881972074508667, "val/ratio_var": 7.150996680138633e-05, "val/num_eos_tokens": 82, "lr": 5e-06, "episode": 512, "epoch": 0.02585858585858586, "step": 32}, {"eps": 0, "objective/kl": 3.579554557800293, "objective/entropy": -73.17301177978516, "objective/non_score_reward": -0.17897772789001465, "objective/rlhf_reward": 7.766334533691406, "objective/scores": 7.9375, "policy/approxkl_avg": 0.0008378748898394406, "policy/clipfrac_avg": 0.0022445092909038067, "loss/policy_avg": 0.09306781738996506, "loss/value_avg": 3.8982434272766113, "val/clipfrac_avg": 0.09437251091003418, "policy/entropy_avg": 0.22289752960205078, "val/ratio": 0.9860420227050781, "val/ratio_var": 6.518009467981756e-05, "val/num_eos_tokens": 388, "lr": 5.161290322580646e-06, "episode": 528, "epoch": 0.02666666666666667, "step": 33}, {"eps": 0, "objective/kl": 5.92071533203125, "objective/entropy": -104.75776672363281, "objective/non_score_reward": -0.2960357964038849, "objective/rlhf_reward": 8.387557983398438, "objective/scores": 8.6875, "policy/approxkl_avg": 0.001011505490168929, "policy/clipfrac_avg": 0.004968347493559122, "loss/policy_avg": -0.2350362241268158, "loss/value_avg": 5.297033309936523, "val/clipfrac_avg": 0.33360281586647034, "policy/entropy_avg": 0.22486305236816406, "val/ratio": 0.9869726300239563, "val/ratio_var": 0.00013502036745194346, "val/num_eos_tokens": 871, "lr": 5.322580645161291e-06, "episode": 544, "epoch": 0.027474747474747475, "step": 34}, {"eps": 0, "objective/kl": 5.674022674560547, "objective/entropy": -92.61628723144531, "objective/non_score_reward": -0.2837011516094208, "objective/rlhf_reward": 6.544424057006836, "objective/scores": 6.8125, "policy/approxkl_avg": 0.0009126091026701033, "policy/clipfrac_avg": 0.005444251000881195, "loss/policy_avg": 0.01827317103743553, "loss/value_avg": 2.6717967987060547, "val/clipfrac_avg": 0.19933006167411804, "policy/entropy_avg": 0.2899208068847656, "val/ratio": 0.9873366355895996, "val/ratio_var": 0.00010928975098067895, "val/num_eos_tokens": 494, "lr": 5.483870967741935e-06, "episode": 560, "epoch": 0.028282828282828285, "step": 35}, {"eps": 0, "objective/kl": 5.049452304840088, "objective/entropy": -116.77957916259766, "objective/non_score_reward": -0.2524726092815399, "objective/rlhf_reward": 11.341278076171875, "objective/scores": 11.625, "policy/approxkl_avg": 0.0007875356823205948, "policy/clipfrac_avg": 0.002243435475975275, "loss/policy_avg": -0.45309978723526, "loss/value_avg": 7.705583572387695, "val/clipfrac_avg": 0.41723158955574036, "policy/entropy_avg": 0.1687488555908203, "val/ratio": 0.9891025424003601, "val/ratio_var": 7.570298475911841e-05, "val/num_eos_tokens": 472, "lr": 5.645161290322582e-06, "episode": 576, "epoch": 0.02909090909090909, "step": 36}, {"eps": 0, "objective/kl": 6.886253833770752, "objective/entropy": -141.64932250976562, "objective/non_score_reward": -0.3443126678466797, "objective/rlhf_reward": 8.91349983215332, "objective/scores": 9.25, "policy/approxkl_avg": 0.0009149812394753098, "policy/clipfrac_avg": 0.0036448687314987183, "loss/policy_avg": -0.27262574434280396, "loss/value_avg": 4.820296287536621, "val/clipfrac_avg": 0.12755423784255981, "policy/entropy_avg": 0.20346355438232422, "val/ratio": 0.9880065321922302, "val/ratio_var": 0.00010917898180196062, "val/num_eos_tokens": 163, "lr": 5.806451612903226e-06, "episode": 592, "epoch": 0.0298989898989899, "step": 37}, {"eps": 0, "objective/kl": 4.568058967590332, "objective/entropy": -68.97830963134766, "objective/non_score_reward": -0.2284029722213745, "objective/rlhf_reward": 8.593862533569336, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0009708706638775766, "policy/clipfrac_avg": 0.003779984312132001, "loss/policy_avg": -0.34780561923980713, "loss/value_avg": 6.757951736450195, "val/clipfrac_avg": 0.19302648305892944, "policy/entropy_avg": 0.23283910751342773, "val/ratio": 0.9863685369491577, "val/ratio_var": 0.00010391126124886796, "val/num_eos_tokens": 504, "lr": 5.967741935483872e-06, "episode": 608, "epoch": 0.030707070707070707, "step": 38}, {"eps": 0, "objective/kl": 4.91707181930542, "objective/entropy": -96.96963500976562, "objective/non_score_reward": -0.24585360288619995, "objective/rlhf_reward": 8.472895622253418, "objective/scores": 8.75, "policy/approxkl_avg": 0.0008412165334448218, "policy/clipfrac_avg": 0.0068719531409442425, "loss/policy_avg": -0.19798313081264496, "loss/value_avg": 3.1714487075805664, "val/clipfrac_avg": 0.14115071296691895, "policy/entropy_avg": 0.1824016571044922, "val/ratio": 0.9887563586235046, "val/ratio_var": 7.865983934607357e-05, "val/num_eos_tokens": 150, "lr": 6.129032258064517e-06, "episode": 624, "epoch": 0.03151515151515152, "step": 39}, {"eps": 0, "objective/kl": 6.146427631378174, "objective/entropy": -71.3642807006836, "objective/non_score_reward": -0.3073214292526245, "objective/rlhf_reward": 8.700490951538086, "objective/scores": 9.0, "policy/approxkl_avg": 0.0008069784380495548, "policy/clipfrac_avg": 0.004310991615056992, "loss/policy_avg": -0.18319812417030334, "loss/value_avg": 2.141171455383301, "val/clipfrac_avg": 0.11660860478878021, "policy/entropy_avg": 0.1599130630493164, "val/ratio": 0.9895697832107544, "val/ratio_var": 3.526661384967156e-05, "val/num_eos_tokens": 20, "lr": 6.290322580645162e-06, "episode": 640, "epoch": 0.03232323232323232, "step": 40}, {"eps": 0, "objective/kl": 6.445234775543213, "objective/entropy": -72.12484741210938, "objective/non_score_reward": -0.3222617506980896, "objective/rlhf_reward": 9.904300689697266, "objective/scores": 10.25, "policy/approxkl_avg": 0.001101147965528071, "policy/clipfrac_avg": 0.005892510525882244, "loss/policy_avg": -0.050840627402067184, "loss/value_avg": 3.107524871826172, "val/clipfrac_avg": 0.28902360796928406, "policy/entropy_avg": 0.23779296875, "val/ratio": 0.9857878684997559, "val/ratio_var": 8.489257743349299e-05, "val/num_eos_tokens": 54, "lr": 6.451612903225806e-06, "episode": 656, "epoch": 0.03313131313131313, "step": 41}, {"eps": 0, "objective/kl": 7.493732452392578, "objective/entropy": -90.39147186279297, "objective/non_score_reward": -0.3746866285800934, "objective/rlhf_reward": 6.710762977600098, "objective/scores": 7.09375, "policy/approxkl_avg": 0.0008206125348806381, "policy/clipfrac_avg": 0.002678059274330735, "loss/policy_avg": -0.12600231170654297, "loss/value_avg": 4.626214981079102, "val/clipfrac_avg": 0.11169028282165527, "policy/entropy_avg": 0.19233226776123047, "val/ratio": 0.988521933555603, "val/ratio_var": 8.11065110610798e-05, "val/num_eos_tokens": 8, "lr": 6.612903225806452e-06, "episode": 672, "epoch": 0.03393939393939394, "step": 42}, {"eps": 0, "objective/kl": 5.224013328552246, "objective/entropy": -97.91592407226562, "objective/non_score_reward": -0.2612006962299347, "objective/rlhf_reward": 9.063018798828125, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0010877563618123531, "policy/clipfrac_avg": 0.004649983253329992, "loss/policy_avg": -0.43953514099121094, "loss/value_avg": 6.373354911804199, "val/clipfrac_avg": 0.22453558444976807, "policy/entropy_avg": 0.2086353302001953, "val/ratio": 0.985620379447937, "val/ratio_var": 0.0001359783491352573, "val/num_eos_tokens": 302, "lr": 6.774193548387097e-06, "episode": 688, "epoch": 0.03474747474747475, "step": 43}, {"eps": 0, "objective/kl": 8.199766159057617, "objective/entropy": -113.148681640625, "objective/non_score_reward": -0.4099883437156677, "objective/rlhf_reward": 6.9806365966796875, "objective/scores": 7.375, "policy/approxkl_avg": 0.0009478515712544322, "policy/clipfrac_avg": 0.002450365573167801, "loss/policy_avg": -0.2054598033428192, "loss/value_avg": 3.1007838249206543, "val/clipfrac_avg": 0.11797119677066803, "policy/entropy_avg": 0.21928471326828003, "val/ratio": 0.9870466589927673, "val/ratio_var": 9.353987115900964e-05, "val/num_eos_tokens": 408, "lr": 6.935483870967743e-06, "episode": 704, "epoch": 0.035555555555555556, "step": 44}, {"eps": 0, "objective/kl": 6.225659370422363, "objective/entropy": -61.80511474609375, "objective/non_score_reward": -0.3112829923629761, "objective/rlhf_reward": 7.524654388427734, "objective/scores": 7.84375, "policy/approxkl_avg": 0.0009408287587575614, "policy/clipfrac_avg": 0.00496808672323823, "loss/policy_avg": 0.06887191534042358, "loss/value_avg": 1.435150384902954, "val/clipfrac_avg": 0.1740478277206421, "policy/entropy_avg": 0.19115066528320312, "val/ratio": 0.9886248111724854, "val/ratio_var": 4.797361543751322e-05, "val/num_eos_tokens": 165, "lr": 7.096774193548388e-06, "episode": 720, "epoch": 0.03636363636363636, "step": 45}, {"eps": 0, "objective/kl": 9.153085708618164, "objective/entropy": -80.8755111694336, "objective/non_score_reward": -0.45765426754951477, "objective/rlhf_reward": 7.393908500671387, "objective/scores": 7.84375, "policy/approxkl_avg": 0.0009702641982585192, "policy/clipfrac_avg": 0.0038732639513909817, "loss/policy_avg": -0.008727293461561203, "loss/value_avg": 1.2011616230010986, "val/clipfrac_avg": 0.1519191861152649, "policy/entropy_avg": 0.20413970947265625, "val/ratio": 0.9864258170127869, "val/ratio_var": 5.902982229599729e-05, "val/num_eos_tokens": 330, "lr": 7.258064516129033e-06, "episode": 736, "epoch": 0.037171717171717175, "step": 46}, {"eps": 0, "objective/kl": 7.737156867980957, "objective/entropy": -74.29957580566406, "objective/non_score_reward": -0.38685786724090576, "objective/rlhf_reward": 7.066267013549805, "objective/scores": 7.4375, "policy/approxkl_avg": 0.0008943852735683322, "policy/clipfrac_avg": 0.007206573151051998, "loss/policy_avg": -0.009945835918188095, "loss/value_avg": 1.0838792324066162, "val/clipfrac_avg": 0.15996162593364716, "policy/entropy_avg": 0.1891345977783203, "val/ratio": 0.9879242181777954, "val/ratio_var": 4.614832141669467e-05, "val/num_eos_tokens": 539, "lr": 7.4193548387096784e-06, "episode": 752, "epoch": 0.03797979797979798, "step": 47}, {"eps": 0, "objective/kl": 4.253904819488525, "objective/entropy": -73.60452270507812, "objective/non_score_reward": -0.21269527077674866, "objective/rlhf_reward": 7.465039253234863, "objective/scores": 7.6875, "policy/approxkl_avg": 0.001011924003250897, "policy/clipfrac_avg": 0.01086447387933731, "loss/policy_avg": -0.12174442410469055, "loss/value_avg": 4.897049903869629, "val/clipfrac_avg": 0.2783915400505066, "policy/entropy_avg": 0.2534313201904297, "val/ratio": 0.9866998195648193, "val/ratio_var": 0.0001104455950553529, "val/num_eos_tokens": 180, "lr": 7.580645161290323e-06, "episode": 768, "epoch": 0.03878787878787879, "step": 48}, {"eps": 0, "objective/kl": 6.293693542480469, "objective/entropy": -61.99099349975586, "objective/non_score_reward": -0.3146846890449524, "objective/rlhf_reward": 7.892346382141113, "objective/scores": 8.1875, "policy/approxkl_avg": 0.0010794082190841436, "policy/clipfrac_avg": 0.013865058310329914, "loss/policy_avg": -0.39860063791275024, "loss/value_avg": 3.1691648960113525, "val/clipfrac_avg": 0.16114407777786255, "policy/entropy_avg": 0.21884393692016602, "val/ratio": 0.9858240485191345, "val/ratio_var": 0.00017287657829001546, "val/num_eos_tokens": 13, "lr": 7.741935483870968e-06, "episode": 784, "epoch": 0.039595959595959594, "step": 49}, {"eps": 0, "objective/kl": 6.167643070220947, "objective/entropy": -71.7900390625, "objective/non_score_reward": -0.30838215351104736, "objective/rlhf_reward": 7.576383590698242, "objective/scores": 7.875, "policy/approxkl_avg": 0.000964264792855829, "policy/clipfrac_avg": 0.009008180350065231, "loss/policy_avg": -0.018761441111564636, "loss/value_avg": 1.6952381134033203, "val/clipfrac_avg": 0.14082017540931702, "policy/entropy_avg": 0.24611949920654297, "val/ratio": 0.9882640838623047, "val/ratio_var": 6.893485260661691e-05, "val/num_eos_tokens": 98, "lr": 7.903225806451613e-06, "episode": 800, "epoch": 0.04040404040404041, "step": 50}, {"eps": 0, "objective/kl": 5.655811309814453, "objective/entropy": -79.89970397949219, "objective/non_score_reward": -0.28279057145118713, "objective/rlhf_reward": 7.611741065979004, "objective/scores": 7.90625, "policy/approxkl_avg": 0.0008801341755315661, "policy/clipfrac_avg": 0.007216072641313076, "loss/policy_avg": -0.045411769300699234, "loss/value_avg": 1.3062368631362915, "val/clipfrac_avg": 0.0781465545296669, "policy/entropy_avg": 0.18111515045166016, "val/ratio": 0.9885283708572388, "val/ratio_var": 8.526279998477548e-05, "val/num_eos_tokens": 21, "lr": 8.064516129032258e-06, "episode": 816, "epoch": 0.041212121212121214, "step": 51}, {"eps": 0, "objective/kl": 5.510902404785156, "objective/entropy": -170.48452758789062, "objective/non_score_reward": -0.2755451202392578, "objective/rlhf_reward": 7.029630661010742, "objective/scores": 7.3125, "policy/approxkl_avg": 0.0006108771194703877, "policy/clipfrac_avg": 0.004314820747822523, "loss/policy_avg": -0.14512324333190918, "loss/value_avg": 1.5389004945755005, "val/clipfrac_avg": 0.08058575540781021, "policy/entropy_avg": 0.1314830780029297, "val/ratio": 0.9919232130050659, "val/ratio_var": 4.701551370089874e-05, "val/num_eos_tokens": 337, "lr": 8.225806451612904e-06, "episode": 832, "epoch": 0.04202020202020202, "step": 52}, {"eps": 0, "objective/kl": 6.628139495849609, "objective/entropy": -96.16239166259766, "objective/non_score_reward": -0.33140695095062256, "objective/rlhf_reward": 9.07874870300293, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0008359593921341002, "policy/clipfrac_avg": 0.005661661736667156, "loss/policy_avg": -0.38281670212745667, "loss/value_avg": 3.0069656372070312, "val/clipfrac_avg": 0.1024901270866394, "policy/entropy_avg": 0.1713862419128418, "val/ratio": 0.9893118143081665, "val/ratio_var": 6.531424878630787e-05, "val/num_eos_tokens": 9, "lr": 8.387096774193549e-06, "episode": 848, "epoch": 0.042828282828282827, "step": 53}, {"eps": 0, "objective/kl": 8.291646957397461, "objective/entropy": -79.8026123046875, "objective/non_score_reward": -0.41458237171173096, "objective/rlhf_reward": 9.077606201171875, "objective/scores": 9.5, "policy/approxkl_avg": 0.0009487073984928429, "policy/clipfrac_avg": 0.027981357648968697, "loss/policy_avg": -0.015762103721499443, "loss/value_avg": 1.7061840295791626, "val/clipfrac_avg": 0.12049607932567596, "policy/entropy_avg": 0.22313880920410156, "val/ratio": 0.9876121282577515, "val/ratio_var": 7.097758498275653e-05, "val/num_eos_tokens": 151, "lr": 8.548387096774194e-06, "episode": 864, "epoch": 0.04363636363636364, "step": 54}, {"eps": 0, "objective/kl": 10.053032875061035, "objective/entropy": -34.682456970214844, "objective/non_score_reward": -0.5026516914367676, "objective/rlhf_reward": 9.54031753540039, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0009677361231297255, "policy/clipfrac_avg": 0.0033034570515155792, "loss/policy_avg": -0.33922648429870605, "loss/value_avg": 1.7691230773925781, "val/clipfrac_avg": 0.17926591634750366, "policy/entropy_avg": 0.1980457305908203, "val/ratio": 0.9871523380279541, "val/ratio_var": 4.744104444398545e-05, "val/num_eos_tokens": 45, "lr": 8.70967741935484e-06, "episode": 880, "epoch": 0.044444444444444446, "step": 55}, {"eps": 0, "objective/kl": 7.366510391235352, "objective/entropy": -89.58358764648438, "objective/non_score_reward": -0.36832550168037415, "objective/rlhf_reward": 7.397299289703369, "objective/scores": 7.75, "policy/approxkl_avg": 0.0008684438071213663, "policy/clipfrac_avg": 0.012218687683343887, "loss/policy_avg": -0.1486111581325531, "loss/value_avg": 3.018749713897705, "val/clipfrac_avg": 0.1270667314529419, "policy/entropy_avg": 0.19923949241638184, "val/ratio": 0.9883291721343994, "val/ratio_var": 7.791651296429336e-05, "val/num_eos_tokens": 282, "lr": 8.870967741935484e-06, "episode": 896, "epoch": 0.04525252525252525, "step": 56}, {"eps": 0, "objective/kl": 9.809646606445312, "objective/entropy": -50.028282165527344, "objective/non_score_reward": -0.49048230051994324, "objective/rlhf_reward": 7.368892669677734, "objective/scores": 7.875, "policy/approxkl_avg": 0.0011404260294511914, "policy/clipfrac_avg": 0.009378134272992611, "loss/policy_avg": 0.2920738160610199, "loss/value_avg": 2.8446502685546875, "val/clipfrac_avg": 0.22250550985336304, "policy/entropy_avg": 0.32576751708984375, "val/ratio": 0.9856222867965698, "val/ratio_var": 4.662256105802953e-05, "val/num_eos_tokens": 524, "lr": 9.03225806451613e-06, "episode": 912, "epoch": 0.04606060606060606, "step": 57}, {"eps": 0, "objective/kl": 8.777005195617676, "objective/entropy": -113.26528930664062, "objective/non_score_reward": -0.4388502836227417, "objective/rlhf_reward": 9.412712097167969, "objective/scores": 9.875, "policy/approxkl_avg": 0.0008379302453249693, "policy/clipfrac_avg": 0.012378448620438576, "loss/policy_avg": -0.4045873284339905, "loss/value_avg": 2.1512539386749268, "val/clipfrac_avg": 0.29461991786956787, "policy/entropy_avg": 0.17434120178222656, "val/ratio": 0.988739550113678, "val/ratio_var": 8.456465729977936e-05, "val/num_eos_tokens": 164, "lr": 9.193548387096775e-06, "episode": 928, "epoch": 0.04686868686868687, "step": 58}, {"eps": 0, "objective/kl": 8.044000625610352, "objective/entropy": -101.6314926147461, "objective/non_score_reward": -0.40220004320144653, "objective/rlhf_reward": 8.168111801147461, "objective/scores": 8.5625, "policy/approxkl_avg": 0.009142342954874039, "policy/clipfrac_avg": 0.02357684075832367, "loss/policy_avg": 0.15078622102737427, "loss/value_avg": 1.8014607429504395, "val/clipfrac_avg": 0.2494688183069229, "policy/entropy_avg": 0.19042015075683594, "val/ratio": 0.9888198375701904, "val/ratio_var": 4.30569052696228e-05, "val/num_eos_tokens": 741, "lr": 9.35483870967742e-06, "episode": 944, "epoch": 0.04767676767676768, "step": 59}, {"eps": 0, "objective/kl": 10.100278854370117, "objective/entropy": -105.13027954101562, "objective/non_score_reward": -0.5050139427185059, "objective/rlhf_reward": 8.147329330444336, "objective/scores": 8.625, "policy/approxkl_avg": 0.0012912012171000242, "policy/clipfrac_avg": 0.01893148198723793, "loss/policy_avg": 0.05019530653953552, "loss/value_avg": 2.0129008293151855, "val/clipfrac_avg": 0.12193797528743744, "policy/entropy_avg": 0.21358871459960938, "val/ratio": 0.9880148768424988, "val/ratio_var": 6.140353070804849e-05, "val/num_eos_tokens": 80, "lr": 9.516129032258065e-06, "episode": 960, "epoch": 0.048484848484848485, "step": 60}, {"eps": 0, "objective/kl": 7.407527923583984, "objective/entropy": -46.50836944580078, "objective/non_score_reward": -0.3703763782978058, "objective/rlhf_reward": 9.981185913085938, "objective/scores": 10.375, "policy/approxkl_avg": 0.0010337126441299915, "policy/clipfrac_avg": 0.014061791822314262, "loss/policy_avg": -0.14436417818069458, "loss/value_avg": 2.7013297080993652, "val/clipfrac_avg": 0.17322519421577454, "policy/entropy_avg": 0.18859481811523438, "val/ratio": 0.9879202246665955, "val/ratio_var": 5.444034104584716e-05, "val/num_eos_tokens": 9, "lr": 9.67741935483871e-06, "episode": 976, "epoch": 0.04929292929292929, "step": 61}, {"eps": 0, "objective/kl": 6.459438323974609, "objective/entropy": -104.4605712890625, "objective/non_score_reward": -0.3229719400405884, "objective/rlhf_reward": 10.25124740600586, "objective/scores": 10.5625, "policy/approxkl_avg": 0.0008976229000836611, "policy/clipfrac_avg": 0.00845531839877367, "loss/policy_avg": -0.4379100501537323, "loss/value_avg": 2.9977834224700928, "val/clipfrac_avg": 0.25087857246398926, "policy/entropy_avg": 0.18575620651245117, "val/ratio": 0.9883289337158203, "val/ratio_var": 0.00015057438577059656, "val/num_eos_tokens": 47, "lr": 9.838709677419356e-06, "episode": 992, "epoch": 0.050101010101010104, "step": 62}, {"eps": 0, "objective/kl": 8.997238159179688, "objective/entropy": -74.31800842285156, "objective/non_score_reward": -0.44986194372177124, "objective/rlhf_reward": 8.124356269836426, "objective/scores": 8.5625, "policy/approxkl_avg": 0.0009533560369163752, "policy/clipfrac_avg": 0.01315313670784235, "loss/policy_avg": 0.0116458460688591, "loss/value_avg": 0.6588444709777832, "val/clipfrac_avg": 0.19718438386917114, "policy/entropy_avg": 0.20091533660888672, "val/ratio": 0.987379789352417, "val/ratio_var": 5.34383361809887e-05, "val/num_eos_tokens": 181, "lr": 1e-05, "episode": 1008, "epoch": 0.05090909090909091, "step": 63}, {"eps": 0, "objective/kl": 9.876341819763184, "objective/entropy": -115.92865753173828, "objective/non_score_reward": -0.4938170909881592, "objective/rlhf_reward": 8.553058624267578, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0009387531317770481, "policy/clipfrac_avg": 0.011313371360301971, "loss/policy_avg": -0.17724251747131348, "loss/value_avg": 2.510040521621704, "val/clipfrac_avg": 0.19327464699745178, "policy/entropy_avg": 0.2193007469177246, "val/ratio": 0.9880937337875366, "val/ratio_var": 6.919403676874936e-05, "val/num_eos_tokens": 433, "lr": 9.999982158767522e-06, "episode": 1024, "epoch": 0.05171717171717172, "step": 64}, {"eps": 0, "objective/kl": 12.105890274047852, "objective/entropy": -151.23590087890625, "objective/non_score_reward": -0.6052945256233215, "objective/rlhf_reward": 7.050955295562744, "objective/scores": 7.65625, "policy/approxkl_avg": 0.0008311746641993523, "policy/clipfrac_avg": 0.021521538496017456, "loss/policy_avg": -0.01736600697040558, "loss/value_avg": 1.1105473041534424, "val/clipfrac_avg": 0.2209428995847702, "policy/entropy_avg": 0.16837120056152344, "val/ratio": 0.9889916777610779, "val/ratio_var": 6.483616743935272e-05, "val/num_eos_tokens": 350, "lr": 9.999928635197409e-06, "episode": 1040, "epoch": 0.052525252525252523, "step": 65}, {"eps": 0, "objective/kl": 9.621443748474121, "objective/entropy": -121.24343872070312, "objective/non_score_reward": -0.48107218742370605, "objective/rlhf_reward": 7.540412425994873, "objective/scores": 8.0, "policy/approxkl_avg": 0.0009060403681360185, "policy/clipfrac_avg": 0.010464828461408615, "loss/policy_avg": -0.16295818984508514, "loss/value_avg": 1.7638646364212036, "val/clipfrac_avg": 0.20382188260555267, "policy/entropy_avg": 0.1780584454536438, "val/ratio": 0.9872162342071533, "val/ratio_var": 0.00010221501725027338, "val/num_eos_tokens": 91, "lr": 9.999839429671632e-06, "episode": 1056, "epoch": 0.05333333333333334, "step": 66}, {"eps": 0, "objective/kl": 9.103609085083008, "objective/entropy": -104.05907440185547, "objective/non_score_reward": -0.45518046617507935, "objective/rlhf_reward": 9.693256378173828, "objective/scores": 10.125, "policy/approxkl_avg": 0.0011231796815991402, "policy/clipfrac_avg": 0.014930792152881622, "loss/policy_avg": 0.15740007162094116, "loss/value_avg": 3.3765735626220703, "val/clipfrac_avg": 0.19888150691986084, "policy/entropy_avg": 0.22347259521484375, "val/ratio": 0.9885534048080444, "val/ratio_var": 0.00011293533316347748, "val/num_eos_tokens": 446, "lr": 9.999714542826806e-06, "episode": 1072, "epoch": 0.05414141414141414, "step": 67}, {"eps": 0, "objective/kl": 11.139631271362305, "objective/entropy": -45.16947555541992, "objective/non_score_reward": -0.5569815635681152, "objective/rlhf_reward": 7.778955936431885, "objective/scores": 8.3125, "policy/approxkl_avg": 0.0012054555118083954, "policy/clipfrac_avg": 0.008249208331108093, "loss/policy_avg": -0.08715461194515228, "loss/value_avg": 0.9404190182685852, "val/clipfrac_avg": 0.24800170958042145, "policy/entropy_avg": 0.19194626808166504, "val/ratio": 0.9870567917823792, "val/ratio_var": 9.06415589270182e-05, "val/num_eos_tokens": 79, "lr": 9.999553975554186e-06, "episode": 1088, "epoch": 0.05494949494949495, "step": 68}, {"eps": 0, "objective/kl": 8.01171588897705, "objective/entropy": -115.63008117675781, "objective/non_score_reward": -0.400585800409317, "objective/rlhf_reward": 10.536914825439453, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0008125576423481107, "policy/clipfrac_avg": 0.011305256746709347, "loss/policy_avg": -0.24446356296539307, "loss/value_avg": 2.7453207969665527, "val/clipfrac_avg": 0.2002556025981903, "policy/entropy_avg": 0.14436888694763184, "val/ratio": 0.9931237697601318, "val/ratio_var": 0.0001478532503824681, "val/num_eos_tokens": 396, "lr": 9.999357728999657e-06, "episode": 1104, "epoch": 0.055757575757575756, "step": 69}, {"eps": 0, "objective/kl": 10.501497268676758, "objective/entropy": -155.4931640625, "objective/non_score_reward": -0.5250749588012695, "objective/rlhf_reward": 9.31476879119873, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0009476456907577813, "policy/clipfrac_avg": 0.008611776866018772, "loss/policy_avg": -0.27069106698036194, "loss/value_avg": 3.394862413406372, "val/clipfrac_avg": 0.258465975522995, "policy/entropy_avg": 0.17398953437805176, "val/ratio": 0.9879174828529358, "val/ratio_var": 0.0001811307593015954, "val/num_eos_tokens": 621, "lr": 9.999125804563732e-06, "episode": 1120, "epoch": 0.05656565656565657, "step": 70}, {"eps": 0, "objective/kl": 11.062005996704102, "objective/entropy": -32.39612579345703, "objective/non_score_reward": -0.5531003475189209, "objective/rlhf_reward": 10.0718994140625, "objective/scores": 10.625, "policy/approxkl_avg": 0.0010651771444827318, "policy/clipfrac_avg": 0.018125716596841812, "loss/policy_avg": 0.06734678149223328, "loss/value_avg": 2.078795909881592, "val/clipfrac_avg": 0.22745996713638306, "policy/entropy_avg": 0.2026987075805664, "val/ratio": 0.9876272678375244, "val/ratio_var": 4.869156327913515e-05, "val/num_eos_tokens": 56, "lr": 9.99885820390154e-06, "episode": 1136, "epoch": 0.057373737373737375, "step": 71}, {"eps": 0, "objective/kl": 8.335628509521484, "objective/entropy": -84.52821350097656, "objective/non_score_reward": -0.4167814254760742, "objective/rlhf_reward": 7.708218574523926, "objective/scores": 8.125, "policy/approxkl_avg": 0.0018768985755741596, "policy/clipfrac_avg": 0.03232686221599579, "loss/policy_avg": 0.360561728477478, "loss/value_avg": 3.147411346435547, "val/clipfrac_avg": 0.21321573853492737, "policy/entropy_avg": 0.30700182914733887, "val/ratio": 0.9917864799499512, "val/ratio_var": 8.937316306401044e-05, "val/num_eos_tokens": 447, "lr": 9.99855492892281e-06, "episode": 1152, "epoch": 0.05818181818181818, "step": 72}, {"eps": 0, "objective/kl": 14.068692207336426, "objective/entropy": -81.507568359375, "objective/non_score_reward": -0.7034346461296082, "objective/rlhf_reward": 8.046566009521484, "objective/scores": 8.75, "policy/approxkl_avg": 0.0029386423993855715, "policy/clipfrac_avg": 0.043007656931877136, "loss/policy_avg": -0.040888212621212006, "loss/value_avg": 1.4504942893981934, "val/clipfrac_avg": 0.07042980194091797, "policy/entropy_avg": 0.19576501846313477, "val/ratio": 0.9879980087280273, "val/ratio_var": 7.574875780846924e-05, "val/num_eos_tokens": 149, "lr": 9.998215981791862e-06, "episode": 1168, "epoch": 0.05898989898989899, "step": 73}, {"eps": 0, "objective/kl": 9.671992301940918, "objective/entropy": -95.04641723632812, "objective/non_score_reward": -0.48359963297843933, "objective/rlhf_reward": 10.774212837219238, "objective/scores": 11.25, "policy/approxkl_avg": 0.0007605117280036211, "policy/clipfrac_avg": 0.009797083213925362, "loss/policy_avg": -0.14120109379291534, "loss/value_avg": 2.754734516143799, "val/clipfrac_avg": 0.1554110050201416, "policy/entropy_avg": 0.148110032081604, "val/ratio": 0.9907310009002686, "val/ratio_var": 6.644838867941871e-05, "val/num_eos_tokens": 261, "lr": 9.997841364927587e-06, "episode": 1184, "epoch": 0.0597979797979798, "step": 74}, {"eps": 0, "objective/kl": 10.376567840576172, "objective/entropy": -133.35256958007812, "objective/non_score_reward": -0.5188283920288086, "objective/rlhf_reward": 5.967499732971191, "objective/scores": 6.5, "policy/approxkl_avg": 0.0010453840950503945, "policy/clipfrac_avg": 0.027546092867851257, "loss/policy_avg": 0.45924872159957886, "loss/value_avg": 2.2186315059661865, "val/clipfrac_avg": 0.2535858154296875, "policy/entropy_avg": 0.23077678680419922, "val/ratio": 0.9903374910354614, "val/ratio_var": 3.860687866108492e-05, "val/num_eos_tokens": 319, "lr": 9.99743108100344e-06, "episode": 1200, "epoch": 0.06060606060606061, "step": 75}, {"eps": 0, "objective/kl": 11.160400390625, "objective/entropy": -95.95547485351562, "objective/non_score_reward": -0.5580199956893921, "objective/rlhf_reward": 9.328699111938477, "objective/scores": 9.875, "policy/approxkl_avg": 0.0008519854163751006, "policy/clipfrac_avg": 0.014135795645415783, "loss/policy_avg": 0.05463395267724991, "loss/value_avg": 1.4657877683639526, "val/clipfrac_avg": 0.11248327791690826, "policy/entropy_avg": 0.1674652099609375, "val/ratio": 0.9904640316963196, "val/ratio_var": 5.162335583008826e-05, "val/num_eos_tokens": 231, "lr": 9.996985132947408e-06, "episode": 1216, "epoch": 0.061414141414141414, "step": 76}, {"eps": 0, "objective/kl": 8.947230339050293, "objective/entropy": -142.799560546875, "objective/non_score_reward": -0.4473615288734436, "objective/rlhf_reward": 8.033106803894043, "objective/scores": 8.5, "policy/approxkl_avg": 0.0007396876462735236, "policy/clipfrac_avg": 0.006773149594664574, "loss/policy_avg": 0.0632552057504654, "loss/value_avg": 1.2827647924423218, "val/clipfrac_avg": 0.07109327614307404, "policy/entropy_avg": 0.16584384441375732, "val/ratio": 0.9911788105964661, "val/ratio_var": 5.5046573834260926e-05, "val/num_eos_tokens": 303, "lr": 9.996503523941994e-06, "episode": 1232, "epoch": 0.06222222222222222, "step": 77}, {"eps": 0, "objective/kl": 9.970010757446289, "objective/entropy": -138.09356689453125, "objective/non_score_reward": -0.4985005855560303, "objective/rlhf_reward": 10.21243667602539, "objective/scores": 10.6875, "policy/approxkl_avg": 0.000907164765521884, "policy/clipfrac_avg": 0.008060376159846783, "loss/policy_avg": -0.7663191556930542, "loss/value_avg": 2.673025608062744, "val/clipfrac_avg": 0.2591398358345032, "policy/entropy_avg": 0.15853583812713623, "val/ratio": 0.9908342361450195, "val/ratio_var": 5.449764648801647e-05, "val/num_eos_tokens": 47, "lr": 9.9959862574242e-06, "episode": 1248, "epoch": 0.06303030303030303, "step": 78}, {"eps": 0, "objective/kl": 8.261874198913574, "objective/entropy": -55.90776062011719, "objective/non_score_reward": -0.4130937457084656, "objective/rlhf_reward": 8.180656433105469, "objective/scores": 8.625, "policy/approxkl_avg": 0.00221771071664989, "policy/clipfrac_avg": 0.018315991386771202, "loss/policy_avg": 0.30310457944869995, "loss/value_avg": 2.481609344482422, "val/clipfrac_avg": 0.29067862033843994, "policy/entropy_avg": 0.19953298568725586, "val/ratio": 0.987062931060791, "val/ratio_var": 7.512731099268422e-05, "val/num_eos_tokens": 16, "lr": 9.995433337085492e-06, "episode": 1264, "epoch": 0.06383838383838383, "step": 79}, {"eps": 0, "objective/kl": 10.453306198120117, "objective/entropy": -114.97703552246094, "objective/non_score_reward": -0.5226653218269348, "objective/rlhf_reward": 8.910928726196289, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0008272784180007875, "policy/clipfrac_avg": 0.008443601429462433, "loss/policy_avg": -0.13517440855503082, "loss/value_avg": 1.4077324867248535, "val/clipfrac_avg": 0.24559587240219116, "policy/entropy_avg": 0.1677337884902954, "val/ratio": 0.9903725981712341, "val/ratio_var": 9.728262375574559e-05, "val/num_eos_tokens": 597, "lr": 9.994844766871785e-06, "episode": 1280, "epoch": 0.06464646464646465, "step": 80}, {"eps": 0, "objective/kl": 8.214365005493164, "objective/entropy": -65.29562377929688, "objective/non_score_reward": -0.41071823239326477, "objective/rlhf_reward": 9.48771858215332, "objective/scores": 9.875, "policy/approxkl_avg": 0.0007956020417623222, "policy/clipfrac_avg": 0.006707598455250263, "loss/policy_avg": -0.22855637967586517, "loss/value_avg": 2.5628278255462646, "val/clipfrac_avg": 0.12181060016155243, "policy/entropy_avg": 0.13605308532714844, "val/ratio": 0.9903298020362854, "val/ratio_var": 5.526099630515091e-05, "val/num_eos_tokens": 276, "lr": 9.994220550983404e-06, "episode": 1296, "epoch": 0.06545454545454546, "step": 81}, {"eps": 0, "objective/kl": 8.709017753601074, "objective/entropy": -71.39965057373047, "objective/non_score_reward": -0.43545088171958923, "objective/rlhf_reward": 7.359470844268799, "objective/scores": 7.78125, "policy/approxkl_avg": 0.0009509660303592682, "policy/clipfrac_avg": 0.01494086068123579, "loss/policy_avg": 0.3731042444705963, "loss/value_avg": 1.8982326984405518, "val/clipfrac_avg": 0.34438806772232056, "policy/entropy_avg": 0.18790435791015625, "val/ratio": 0.9874451160430908, "val/ratio_var": 3.9616661524632946e-05, "val/num_eos_tokens": 276, "lr": 9.993560693875065e-06, "episode": 1312, "epoch": 0.06626262626262626, "step": 82}, {"eps": 0, "objective/kl": 9.877692222595215, "objective/entropy": -186.06890869140625, "objective/non_score_reward": -0.4938845634460449, "objective/rlhf_reward": 7.853771686553955, "objective/scores": 8.375, "policy/approxkl_avg": 0.0008737960597500205, "policy/clipfrac_avg": 0.030746009200811386, "loss/policy_avg": -0.06645295768976212, "loss/value_avg": 1.9543490409851074, "val/clipfrac_avg": 0.21427029371261597, "policy/entropy_avg": 0.27783632278442383, "val/ratio": 0.9895691871643066, "val/ratio_var": 0.0001006032107397914, "val/num_eos_tokens": 434, "lr": 9.992865200255829e-06, "episode": 1328, "epoch": 0.06707070707070707, "step": 83}, {"eps": 0, "objective/kl": 10.45096206665039, "objective/entropy": -112.0244140625, "objective/non_score_reward": -0.5225480794906616, "objective/rlhf_reward": 8.91495132446289, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0010605708230286837, "policy/clipfrac_avg": 0.01699572056531906, "loss/policy_avg": 0.06399939954280853, "loss/value_avg": 1.2927840948104858, "val/clipfrac_avg": 0.06052979826927185, "policy/entropy_avg": 0.15537548065185547, "val/ratio": 0.9891489744186401, "val/ratio_var": 7.55400033085607e-05, "val/num_eos_tokens": 693, "lr": 9.992134075089085e-06, "episode": 1344, "epoch": 0.06787878787878789, "step": 84}, {"eps": 0, "objective/kl": 7.841750621795654, "objective/entropy": -110.17214965820312, "objective/non_score_reward": -0.39208751916885376, "objective/rlhf_reward": 9.154787063598633, "objective/scores": 9.5625, "policy/approxkl_avg": 0.0011510876938700676, "policy/clipfrac_avg": 0.029715364798903465, "loss/policy_avg": 0.05262535810470581, "loss/value_avg": 1.9076130390167236, "val/clipfrac_avg": 0.13365106284618378, "policy/entropy_avg": 0.14819884300231934, "val/ratio": 0.9908526539802551, "val/ratio_var": 6.175693852128461e-05, "val/num_eos_tokens": 172, "lr": 9.991367323592499e-06, "episode": 1360, "epoch": 0.06868686868686869, "step": 85}, {"eps": 0, "objective/kl": 10.169289588928223, "objective/entropy": -125.11438751220703, "objective/non_score_reward": -0.5084644556045532, "objective/rlhf_reward": 7.858722686767578, "objective/scores": 8.375, "policy/approxkl_avg": 0.0008927697199396789, "policy/clipfrac_avg": 0.014697541482746601, "loss/policy_avg": 0.29705333709716797, "loss/value_avg": 2.059380531311035, "val/clipfrac_avg": 0.1657203733921051, "policy/entropy_avg": 0.17176449298858643, "val/ratio": 0.9888587594032288, "val/ratio_var": 8.45455433591269e-05, "val/num_eos_tokens": 262, "lr": 9.99056495123799e-06, "episode": 1376, "epoch": 0.0694949494949495, "step": 86}, {"eps": 0, "objective/kl": 9.372626304626465, "objective/entropy": -110.98075866699219, "objective/non_score_reward": -0.4686313271522522, "objective/rlhf_reward": 9.06261920928955, "objective/scores": 9.5, "policy/approxkl_avg": 0.0009293662733398378, "policy/clipfrac_avg": 0.013371942564845085, "loss/policy_avg": -0.310147762298584, "loss/value_avg": 1.6057798862457275, "val/clipfrac_avg": 0.14167499542236328, "policy/entropy_avg": 0.2098407745361328, "val/ratio": 0.9893955588340759, "val/ratio_var": 0.00011555494711501524, "val/num_eos_tokens": 219, "lr": 9.989726963751683e-06, "episode": 1392, "epoch": 0.0703030303030303, "step": 87}, {"eps": 0, "objective/kl": 8.599248886108398, "objective/entropy": -119.2513198852539, "objective/non_score_reward": -0.4299624264240265, "objective/rlhf_reward": 9.562225341796875, "objective/scores": 10.0, "policy/approxkl_avg": 0.0008632695535197854, "policy/clipfrac_avg": 0.014250310137867928, "loss/policy_avg": -0.36583343148231506, "loss/value_avg": 1.4973384141921997, "val/clipfrac_avg": 0.19998914003372192, "policy/entropy_avg": 0.1634058952331543, "val/ratio": 0.9901472330093384, "val/ratio_var": 9.213508747052401e-05, "val/num_eos_tokens": 380, "lr": 9.98885336711387e-06, "episode": 1408, "epoch": 0.07111111111111111, "step": 88}, {"eps": 0, "objective/kl": 7.493285655975342, "objective/entropy": -137.1684112548828, "objective/non_score_reward": -0.374664306640625, "objective/rlhf_reward": 6.914398193359375, "objective/scores": 7.28125, "policy/approxkl_avg": 0.0010005123913288116, "policy/clipfrac_avg": 0.012202544137835503, "loss/policy_avg": 0.3967055678367615, "loss/value_avg": 2.1769919395446777, "val/clipfrac_avg": 0.22100284695625305, "policy/entropy_avg": 0.13602828979492188, "val/ratio": 0.993422269821167, "val/ratio_var": 0.0001790324313333258, "val/num_eos_tokens": 391, "lr": 9.987944167558963e-06, "episode": 1424, "epoch": 0.07191919191919192, "step": 89}, {"eps": 0, "objective/kl": 9.35251235961914, "objective/entropy": -102.01996612548828, "objective/non_score_reward": -0.46762561798095703, "objective/rlhf_reward": 8.868311882019043, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0007846959633752704, "policy/clipfrac_avg": 0.007588379550725222, "loss/policy_avg": -0.166255384683609, "loss/value_avg": 0.6386685967445374, "val/clipfrac_avg": 0.26963502168655396, "policy/entropy_avg": 0.18400812149047852, "val/ratio": 0.9904755353927612, "val/ratio_var": 4.9346734158461913e-05, "val/num_eos_tokens": 381, "lr": 9.986999371575465e-06, "episode": 1440, "epoch": 0.07272727272727272, "step": 90}, {"eps": 0, "objective/kl": 6.7079644203186035, "objective/entropy": -52.68136215209961, "objective/non_score_reward": -0.33539825677871704, "objective/rlhf_reward": 10.49272632598877, "objective/scores": 10.8125, "policy/approxkl_avg": 0.0012186269741505384, "policy/clipfrac_avg": 0.006110272370278835, "loss/policy_avg": 0.06461673974990845, "loss/value_avg": 1.8350507020950317, "val/clipfrac_avg": 0.15903371572494507, "policy/entropy_avg": 0.21361064910888672, "val/ratio": 0.9849614500999451, "val/ratio_var": 0.00012401219282764941, "val/num_eos_tokens": 142, "lr": 9.986018985905901e-06, "episode": 1456, "epoch": 0.07353535353535354, "step": 91}, {"eps": 0, "objective/kl": 11.082052230834961, "objective/entropy": -110.07864379882812, "objective/non_score_reward": -0.5541026592254639, "objective/rlhf_reward": 8.570897102355957, "objective/scores": 9.125, "policy/approxkl_avg": 0.0008647169452160597, "policy/clipfrac_avg": 0.01752229407429695, "loss/policy_avg": 0.2874661684036255, "loss/value_avg": 1.235567569732666, "val/clipfrac_avg": 0.11287079006433487, "policy/entropy_avg": 0.1840806007385254, "val/ratio": 0.9882228374481201, "val/ratio_var": 6.303517875494435e-05, "val/num_eos_tokens": 48, "lr": 9.985003017546789e-06, "episode": 1472, "epoch": 0.07434343434343435, "step": 92}, {"eps": 0, "objective/kl": 10.317655563354492, "objective/entropy": -110.14735412597656, "objective/non_score_reward": -0.5158827900886536, "objective/rlhf_reward": 10.44505500793457, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0007985912961885333, "policy/clipfrac_avg": 0.0036714496091008186, "loss/policy_avg": -1.0834553241729736, "loss/value_avg": 2.374274969100952, "val/clipfrac_avg": 0.26723405718803406, "policy/entropy_avg": 0.1591578722000122, "val/ratio": 0.989249587059021, "val/ratio_var": 6.81774690747261e-05, "val/num_eos_tokens": 348, "lr": 9.983951473748579e-06, "episode": 1488, "epoch": 0.07515151515151515, "step": 93}, {"eps": 0, "objective/kl": 11.676578521728516, "objective/entropy": -26.387767791748047, "objective/non_score_reward": -0.5838289260864258, "objective/rlhf_reward": 8.232576370239258, "objective/scores": 8.8125, "policy/approxkl_avg": 0.00300201796926558, "policy/clipfrac_avg": 0.014051968231797218, "loss/policy_avg": -0.016785837709903717, "loss/value_avg": 2.52266788482666, "val/clipfrac_avg": 0.19883441925048828, "policy/entropy_avg": 0.23279857635498047, "val/ratio": 0.9866381883621216, "val/ratio_var": 6.660662620561197e-05, "val/num_eos_tokens": 74, "lr": 9.982864362015606e-06, "episode": 1504, "epoch": 0.07595959595959596, "step": 94}, {"eps": 0, "objective/kl": 11.7849760055542, "objective/entropy": -111.26895141601562, "objective/non_score_reward": -0.589248776435852, "objective/rlhf_reward": 9.957626342773438, "objective/scores": 10.5625, "policy/approxkl_avg": 0.0032595903612673283, "policy/clipfrac_avg": 0.025392604991793633, "loss/policy_avg": -0.1366100311279297, "loss/value_avg": 2.2388007640838623, "val/clipfrac_avg": 0.1502402424812317, "policy/entropy_avg": 0.23312854766845703, "val/ratio": 0.9824517965316772, "val/ratio_var": 0.0003387684701010585, "val/num_eos_tokens": 15, "lr": 9.981741690106035e-06, "episode": 1520, "epoch": 0.07676767676767676, "step": 95}, {"eps": 0, "objective/kl": 9.119251251220703, "objective/entropy": -67.97806549072266, "objective/non_score_reward": -0.45596253871917725, "objective/rlhf_reward": 7.071381092071533, "objective/scores": 7.53125, "policy/approxkl_avg": 0.0009370450279675424, "policy/clipfrac_avg": 0.023228613659739494, "loss/policy_avg": 0.31284308433532715, "loss/value_avg": 3.7777535915374756, "val/clipfrac_avg": 0.30271416902542114, "policy/entropy_avg": 0.19547653198242188, "val/ratio": 0.9877786636352539, "val/ratio_var": 7.666354213142768e-05, "val/num_eos_tokens": 180, "lr": 9.980583466031808e-06, "episode": 1536, "epoch": 0.07757575757575758, "step": 96}, {"eps": 0, "objective/kl": 10.928716659545898, "objective/entropy": -168.7757568359375, "objective/non_score_reward": -0.5464357733726501, "objective/rlhf_reward": 7.793407917022705, "objective/scores": 8.3125, "policy/approxkl_avg": 0.0010802671313285828, "policy/clipfrac_avg": 0.013121331110596657, "loss/policy_avg": -0.260562539100647, "loss/value_avg": 1.9976837635040283, "val/clipfrac_avg": 0.17170178890228271, "policy/entropy_avg": 0.14757728576660156, "val/ratio": 0.992108166217804, "val/ratio_var": 0.00020361572387628257, "val/num_eos_tokens": 530, "lr": 9.979389698058581e-06, "episode": 1552, "epoch": 0.07838383838383839, "step": 97}, {"eps": 0, "objective/kl": 8.581636428833008, "objective/entropy": -88.97151184082031, "objective/non_score_reward": -0.42908182740211487, "objective/rlhf_reward": 7.738887310028076, "objective/scores": 8.1875, "policy/approxkl_avg": 0.0008207992068491876, "policy/clipfrac_avg": 0.014891392551362514, "loss/policy_avg": 0.20252731442451477, "loss/value_avg": 1.8869277238845825, "val/clipfrac_avg": 0.14818435907363892, "policy/entropy_avg": 0.24853515625, "val/ratio": 0.9901211261749268, "val/ratio_var": 6.446534825954586e-05, "val/num_eos_tokens": 287, "lr": 9.978160394705669e-06, "episode": 1568, "epoch": 0.07919191919191919, "step": 98}, {"eps": 0, "objective/kl": 10.816564559936523, "objective/entropy": -70.99946594238281, "objective/non_score_reward": -0.540828287601471, "objective/rlhf_reward": 8.123233795166016, "objective/scores": 8.6875, "policy/approxkl_avg": 0.002382381819188595, "policy/clipfrac_avg": 0.03098738193511963, "loss/policy_avg": 0.4129256010055542, "loss/value_avg": 1.6777870655059814, "val/clipfrac_avg": 0.17884066700935364, "policy/entropy_avg": 0.19532859325408936, "val/ratio": 0.9863871932029724, "val/ratio_var": 0.00017909934103954583, "val/num_eos_tokens": 287, "lr": 9.976895564745993e-06, "episode": 1584, "epoch": 0.08, "step": 99}, {"eps": 0, "objective/kl": 10.959748268127441, "objective/entropy": -94.05448913574219, "objective/non_score_reward": -0.5479874610900879, "objective/rlhf_reward": 9.295762062072754, "objective/scores": 9.875, "policy/approxkl_avg": 0.0010338419815525413, "policy/clipfrac_avg": 0.009063776582479477, "loss/policy_avg": 0.07570860534906387, "loss/value_avg": 0.5786488652229309, "val/clipfrac_avg": 0.13884295523166656, "policy/entropy_avg": 0.19421935081481934, "val/ratio": 0.9873470664024353, "val/ratio_var": 7.10823223926127e-05, "val/num_eos_tokens": 14, "lr": 9.975595217205995e-06, "episode": 1600, "epoch": 0.08080808080808081, "step": 100}, {"eps": 0, "objective/kl": 9.482419967651367, "objective/entropy": -147.2283172607422, "objective/non_score_reward": -0.4741210341453552, "objective/rlhf_reward": 9.45361328125, "objective/scores": 9.9375, "policy/approxkl_avg": 0.000866247049998492, "policy/clipfrac_avg": 0.011027923785150051, "loss/policy_avg": 0.18178647756576538, "loss/value_avg": 3.246936559677124, "val/clipfrac_avg": 0.21203863620758057, "policy/entropy_avg": 0.18662023544311523, "val/ratio": 0.989406406879425, "val/ratio_var": 9.00850209291093e-05, "val/num_eos_tokens": 62, "lr": 9.974259361365604e-06, "episode": 1616, "epoch": 0.08161616161616161, "step": 101}, {"eps": 0, "objective/kl": 11.228017807006836, "objective/entropy": -102.03694915771484, "objective/non_score_reward": -0.5614008903503418, "objective/rlhf_reward": 9.1944580078125, "objective/scores": 9.75, "policy/approxkl_avg": 0.0009244791581295431, "policy/clipfrac_avg": 0.02670031040906906, "loss/policy_avg": -0.3622581660747528, "loss/value_avg": 3.517216205596924, "val/clipfrac_avg": 0.22908943891525269, "policy/entropy_avg": 0.19944334030151367, "val/ratio": 0.9883830547332764, "val/ratio_var": 8.266136137535796e-05, "val/num_eos_tokens": 360, "lr": 9.97288800675814e-06, "episode": 1632, "epoch": 0.08242424242424243, "step": 102}, {"eps": 0, "objective/kl": 11.77578067779541, "objective/entropy": -142.39923095703125, "objective/non_score_reward": -0.5887890458106995, "objective/rlhf_reward": 9.028398513793945, "objective/scores": 9.625, "policy/approxkl_avg": 0.000976751558482647, "policy/clipfrac_avg": 0.013522524386644363, "loss/policy_avg": 0.004343941807746887, "loss/value_avg": 3.0538196563720703, "val/clipfrac_avg": 0.21960067749023438, "policy/entropy_avg": 0.26236867904663086, "val/ratio": 0.9883174896240234, "val/ratio_var": 0.00010564823605818674, "val/num_eos_tokens": 9, "lr": 9.97148116317027e-06, "episode": 1648, "epoch": 0.08323232323232323, "step": 103}, {"eps": 0, "objective/kl": 12.531501770019531, "objective/entropy": -71.44817352294922, "objective/non_score_reward": -0.6265751123428345, "objective/rlhf_reward": 8.525768280029297, "objective/scores": 9.125, "policy/approxkl_avg": 0.0010512301232665777, "policy/clipfrac_avg": 0.03077285923063755, "loss/policy_avg": 0.4931744933128357, "loss/value_avg": 2.0421762466430664, "val/clipfrac_avg": 0.23763741552829742, "policy/entropy_avg": 0.20035934448242188, "val/ratio": 0.9862467050552368, "val/ratio_var": 0.00010008800018113106, "val/num_eos_tokens": 23, "lr": 9.970038840641918e-06, "episode": 1664, "epoch": 0.08404040404040404, "step": 104}, {"eps": 0, "objective/kl": 11.113540649414062, "objective/entropy": -60.47146224975586, "objective/non_score_reward": -0.555677056312561, "objective/rlhf_reward": 8.44432258605957, "objective/scores": 9.0, "policy/approxkl_avg": 0.001011199550703168, "policy/clipfrac_avg": 0.016472183167934418, "loss/policy_avg": 0.15791772305965424, "loss/value_avg": 1.9950876235961914, "val/clipfrac_avg": 0.2771484851837158, "policy/entropy_avg": 0.15747523307800293, "val/ratio": 0.9897866249084473, "val/ratio_var": 4.6754288632655516e-05, "val/num_eos_tokens": 26, "lr": 9.968561049466214e-06, "episode": 1680, "epoch": 0.08484848484848485, "step": 105}, {"eps": 0, "objective/kl": 12.041143417358398, "objective/entropy": -109.22718811035156, "objective/non_score_reward": -0.6020572185516357, "objective/rlhf_reward": 9.976067543029785, "objective/scores": 10.5625, "policy/approxkl_avg": 0.001960658235475421, "policy/clipfrac_avg": 0.02854878082871437, "loss/policy_avg": 0.14141756296157837, "loss/value_avg": 1.6132752895355225, "val/clipfrac_avg": 0.16923227906227112, "policy/entropy_avg": 0.14309561252593994, "val/ratio": 0.9895997643470764, "val/ratio_var": 5.035091453464702e-05, "val/num_eos_tokens": 19, "lr": 9.967047800189401e-06, "episode": 1696, "epoch": 0.08565656565656565, "step": 106}, {"eps": 0, "objective/kl": 11.278413772583008, "objective/entropy": -166.47471618652344, "objective/non_score_reward": -0.5639207363128662, "objective/rlhf_reward": 11.272016525268555, "objective/scores": 11.8125, "policy/approxkl_avg": 0.0008130659116432071, "policy/clipfrac_avg": 0.009939244017004967, "loss/policy_avg": -0.13957123458385468, "loss/value_avg": 0.8985839486122131, "val/clipfrac_avg": 0.10860806703567505, "policy/entropy_avg": 0.13879013061523438, "val/ratio": 0.9910627603530884, "val/ratio_var": 4.761793388752267e-05, "val/num_eos_tokens": 232, "lr": 9.965499103610775e-06, "episode": 1712, "epoch": 0.08646464646464647, "step": 107}, {"eps": 0, "objective/kl": 10.213882446289062, "objective/entropy": -81.45556640625, "objective/non_score_reward": -0.510694146156311, "objective/rlhf_reward": 9.04399299621582, "objective/scores": 9.5625, "policy/approxkl_avg": 0.0012008333578705788, "policy/clipfrac_avg": 0.010314267128705978, "loss/policy_avg": -0.11392714083194733, "loss/value_avg": 1.136710524559021, "val/clipfrac_avg": 0.16719487309455872, "policy/entropy_avg": 0.22310292720794678, "val/ratio": 0.9873642921447754, "val/ratio_var": 0.000118928866868373, "val/num_eos_tokens": 724, "lr": 9.963914970782594e-06, "episode": 1728, "epoch": 0.08727272727272728, "step": 108}, {"eps": 0, "objective/kl": 17.010793685913086, "objective/entropy": -23.411705017089844, "objective/non_score_reward": -0.8505396842956543, "objective/rlhf_reward": 6.887741565704346, "objective/scores": 7.75, "policy/approxkl_avg": 0.0017146767349913716, "policy/clipfrac_avg": 0.007722174748778343, "loss/policy_avg": 0.02373083308339119, "loss/value_avg": 0.7551393508911133, "val/clipfrac_avg": 0.265152245759964, "policy/entropy_avg": 0.24081039428710938, "val/ratio": 0.9839531779289246, "val/ratio_var": 0.00013079243944957852, "val/num_eos_tokens": 92, "lr": 9.962295413010013e-06, "episode": 1744, "epoch": 0.08808080808080808, "step": 109}, {"eps": 0, "objective/kl": 17.350746154785156, "objective/entropy": -49.22315216064453, "objective/non_score_reward": -0.867537260055542, "objective/rlhf_reward": 7.573868751525879, "objective/scores": 8.4375, "policy/approxkl_avg": 0.001081926515325904, "policy/clipfrac_avg": 0.012329247780144215, "loss/policy_avg": -0.029323376715183258, "loss/value_avg": 0.6305721998214722, "val/clipfrac_avg": 0.14187291264533997, "policy/entropy_avg": 0.19385910034179688, "val/ratio": 0.9876461625099182, "val/ratio_var": 5.4690557590220124e-05, "val/num_eos_tokens": 469, "lr": 9.960640441850997e-06, "episode": 1760, "epoch": 0.08888888888888889, "step": 110}, {"eps": 0, "objective/kl": 13.64469051361084, "objective/entropy": -99.91004180908203, "objective/non_score_reward": -0.682234525680542, "objective/rlhf_reward": 9.591202735900879, "objective/scores": 10.25, "policy/approxkl_avg": 0.0013218795647844672, "policy/clipfrac_avg": 0.023597117513418198, "loss/policy_avg": -0.09420577436685562, "loss/value_avg": 1.1037640571594238, "val/clipfrac_avg": 0.16007894277572632, "policy/entropy_avg": 0.19385337829589844, "val/ratio": 0.9876752495765686, "val/ratio_var": 9.703996329335496e-05, "val/num_eos_tokens": 23, "lr": 9.95895006911623e-06, "episode": 1776, "epoch": 0.08969696969696969, "step": 111}, {"eps": 0, "objective/kl": 15.633418083190918, "objective/entropy": -43.240875244140625, "objective/non_score_reward": -0.7816709280014038, "objective/rlhf_reward": 7.913640975952148, "objective/scores": 8.6875, "policy/approxkl_avg": 0.002886019414290786, "policy/clipfrac_avg": 0.012701047584414482, "loss/policy_avg": -0.012303899973630905, "loss/value_avg": 1.2762086391448975, "val/clipfrac_avg": 0.23217324912548065, "policy/entropy_avg": 0.23091506958007812, "val/ratio": 0.9861587285995483, "val/ratio_var": 0.0003747409500647336, "val/num_eos_tokens": 18, "lr": 9.957224306869053e-06, "episode": 1792, "epoch": 0.0905050505050505, "step": 112}, {"eps": 0, "objective/kl": 12.853160858154297, "objective/entropy": -69.88031768798828, "objective/non_score_reward": -0.6426580548286438, "objective/rlhf_reward": 9.568279266357422, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0011324469232931733, "policy/clipfrac_avg": 0.004497843328863382, "loss/policy_avg": -0.5104568004608154, "loss/value_avg": 2.1639163494110107, "val/clipfrac_avg": 0.28476184606552124, "policy/entropy_avg": 0.16321039199829102, "val/ratio": 0.9893928170204163, "val/ratio_var": 6.093212141422555e-05, "val/num_eos_tokens": 59, "lr": 9.95546316742535e-06, "episode": 1808, "epoch": 0.09131313131313132, "step": 113}, {"eps": 0, "objective/kl": 13.041144371032715, "objective/entropy": -92.63809204101562, "objective/non_score_reward": -0.6520571708679199, "objective/rlhf_reward": 10.613567352294922, "objective/scores": 11.25, "policy/approxkl_avg": 0.0009989608079195023, "policy/clipfrac_avg": 0.012498335912823677, "loss/policy_avg": -0.5700721740722656, "loss/value_avg": 3.390981674194336, "val/clipfrac_avg": 0.16689905524253845, "policy/entropy_avg": 0.1557478904724121, "val/ratio": 0.9884234666824341, "val/ratio_var": 0.0001190627517644316, "val/num_eos_tokens": 337, "lr": 9.953666663353485e-06, "episode": 1824, "epoch": 0.09212121212121212, "step": 114}, {"eps": 0, "objective/kl": 11.188087463378906, "objective/entropy": -62.3432502746582, "objective/non_score_reward": -0.5594043731689453, "objective/rlhf_reward": 9.198408126831055, "objective/scores": 9.75, "policy/approxkl_avg": 0.0008078927057795227, "policy/clipfrac_avg": 0.00809546373784542, "loss/policy_avg": -0.037001632153987885, "loss/value_avg": 1.2843832969665527, "val/clipfrac_avg": 0.14679771661758423, "policy/entropy_avg": 0.16070127487182617, "val/ratio": 0.9907799363136292, "val/ratio_var": 6.536444561788812e-05, "val/num_eos_tokens": 491, "lr": 9.951834807474191e-06, "episode": 1840, "epoch": 0.09292929292929293, "step": 115}, {"eps": 0, "objective/kl": 13.151895523071289, "objective/entropy": -67.57776641845703, "objective/non_score_reward": -0.6575947999954224, "objective/rlhf_reward": 7.943967819213867, "objective/scores": 8.625, "policy/approxkl_avg": 0.001524624414741993, "policy/clipfrac_avg": 0.03400165215134621, "loss/policy_avg": 0.392054945230484, "loss/value_avg": 4.038256645202637, "val/clipfrac_avg": 0.24824200570583344, "policy/entropy_avg": 0.1723041534423828, "val/ratio": 0.989495038986206, "val/ratio_var": 9.145861986326054e-05, "val/num_eos_tokens": 297, "lr": 9.9499676128605e-06, "episode": 1856, "epoch": 0.09373737373737374, "step": 116}, {"eps": 0, "objective/kl": 8.475647926330566, "objective/entropy": -68.10916900634766, "objective/non_score_reward": -0.4237824082374573, "objective/rlhf_reward": 7.5176239013671875, "objective/scores": 7.9375, "policy/approxkl_avg": 0.0007898050826042891, "policy/clipfrac_avg": 0.006391381844878197, "loss/policy_avg": 0.13758663833141327, "loss/value_avg": 1.6034759283065796, "val/clipfrac_avg": 0.2929156422615051, "policy/entropy_avg": 0.14587292075157166, "val/ratio": 0.9905034303665161, "val/ratio_var": 3.43524880008772e-05, "val/num_eos_tokens": 130, "lr": 9.948065092837631e-06, "episode": 1872, "epoch": 0.09454545454545454, "step": 117}, {"eps": 0, "objective/kl": 13.444223403930664, "objective/entropy": -64.47953033447266, "objective/non_score_reward": -0.6722111701965332, "objective/rlhf_reward": 7.370757579803467, "objective/scores": 8.0625, "policy/approxkl_avg": 0.0009727979195304215, "policy/clipfrac_avg": 0.023482225835323334, "loss/policy_avg": -0.010858863592147827, "loss/value_avg": 0.8126935958862305, "val/clipfrac_avg": 0.15926581621170044, "policy/entropy_avg": 0.17380249500274658, "val/ratio": 0.9887762069702148, "val/ratio_var": 4.845508374273777e-05, "val/num_eos_tokens": 899, "lr": 9.946127260982903e-06, "episode": 1888, "epoch": 0.09535353535353536, "step": 118}, {"eps": 0, "objective/kl": 14.874452590942383, "objective/entropy": -142.527587890625, "objective/non_score_reward": -0.7437226176261902, "objective/rlhf_reward": 7.916433811187744, "objective/scores": 8.6875, "policy/approxkl_avg": 0.00139252794906497, "policy/clipfrac_avg": 0.03409278765320778, "loss/policy_avg": 0.05183769389986992, "loss/value_avg": 0.7353260517120361, "val/clipfrac_avg": 0.14925608038902283, "policy/entropy_avg": 0.21674728393554688, "val/ratio": 0.9891353845596313, "val/ratio_var": 7.120384543668479e-05, "val/num_eos_tokens": 683, "lr": 9.944154131125643e-06, "episode": 1904, "epoch": 0.09616161616161616, "step": 119}, {"eps": 0, "objective/kl": 10.851125717163086, "objective/entropy": -30.89788246154785, "objective/non_score_reward": -0.5425562262535095, "objective/rlhf_reward": 7.843674182891846, "objective/scores": 8.375, "policy/approxkl_avg": 0.0013285963796079159, "policy/clipfrac_avg": 0.021273121237754822, "loss/policy_avg": 0.15797683596611023, "loss/value_avg": 2.7395377159118652, "val/clipfrac_avg": 0.16737978160381317, "policy/entropy_avg": 0.20377635955810547, "val/ratio": 0.9880788326263428, "val/ratio_var": 5.1685652579180896e-05, "val/num_eos_tokens": 345, "lr": 9.942145717347077e-06, "episode": 1920, "epoch": 0.09696969696969697, "step": 120}, {"eps": 0, "objective/kl": 14.510229110717773, "objective/entropy": -218.7469024658203, "objective/non_score_reward": -0.7255114316940308, "objective/rlhf_reward": 8.21198844909668, "objective/scores": 8.9375, "policy/approxkl_avg": 0.001183751504868269, "policy/clipfrac_avg": 0.045801788568496704, "loss/policy_avg": 0.22806939482688904, "loss/value_avg": 2.743255615234375, "val/clipfrac_avg": 0.14798504114151, "policy/entropy_avg": 0.18555426597595215, "val/ratio": 0.9906567931175232, "val/ratio_var": 7.052164437482134e-05, "val/num_eos_tokens": 350, "lr": 9.940102033980236e-06, "episode": 1936, "epoch": 0.09777777777777778, "step": 121}, {"eps": 0, "objective/kl": 10.68556022644043, "objective/entropy": -61.34413528442383, "objective/non_score_reward": -0.5342780351638794, "objective/rlhf_reward": 9.78994083404541, "objective/scores": 10.3125, "policy/approxkl_avg": 0.0008475382928736508, "policy/clipfrac_avg": 0.007006281986832619, "loss/policy_avg": -0.28502777218818665, "loss/value_avg": 1.3561985492706299, "val/clipfrac_avg": 0.125126451253891, "policy/entropy_avg": 0.19134902954101562, "val/ratio": 0.9898977279663086, "val/ratio_var": 6.245759141165763e-05, "val/num_eos_tokens": 19, "lr": 9.93802309560985e-06, "episode": 1952, "epoch": 0.09858585858585858, "step": 122}, {"eps": 0, "objective/kl": 11.469106674194336, "objective/entropy": -31.413511276245117, "objective/non_score_reward": -0.5734553337097168, "objective/rlhf_reward": 10.352326393127441, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0017750051338225603, "policy/clipfrac_avg": 0.019242165610194206, "loss/policy_avg": -0.05578336864709854, "loss/value_avg": 3.2866952419281006, "val/clipfrac_avg": 0.1343041956424713, "policy/entropy_avg": 0.23152542114257812, "val/ratio": 0.9851062297821045, "val/ratio_var": 0.00021594094869215041, "val/num_eos_tokens": 111, "lr": 9.935908917072253e-06, "episode": 1968, "epoch": 0.0993939393939394, "step": 123}, {"eps": 0, "objective/kl": 11.678960800170898, "objective/entropy": -41.85675811767578, "objective/non_score_reward": -0.583948016166687, "objective/rlhf_reward": 8.416051864624023, "objective/scores": 9.0, "policy/approxkl_avg": 0.001134565332904458, "policy/clipfrac_avg": 0.009194762445986271, "loss/policy_avg": 0.07291892915964127, "loss/value_avg": 2.094268321990967, "val/clipfrac_avg": 0.20579089224338531, "policy/entropy_avg": 0.22137069702148438, "val/ratio": 0.9855261445045471, "val/ratio_var": 0.00015781399270053953, "val/num_eos_tokens": 25, "lr": 9.93375951345526e-06, "episode": 1984, "epoch": 0.10020202020202021, "step": 124}, {"eps": 0, "objective/kl": 12.147377014160156, "objective/entropy": -38.61683654785156, "objective/non_score_reward": -0.6073689460754395, "objective/rlhf_reward": 8.478569030761719, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0010453371796756983, "policy/clipfrac_avg": 0.02015816420316696, "loss/policy_avg": -0.20081275701522827, "loss/value_avg": 0.9789823293685913, "val/clipfrac_avg": 0.1644173264503479, "policy/entropy_avg": 0.20670151710510254, "val/ratio": 0.9876101016998291, "val/ratio_var": 5.8746740251081064e-05, "val/num_eos_tokens": 31, "lr": 9.931574900098076e-06, "episode": 2000, "epoch": 0.10101010101010101, "step": 125}, {"eps": 0, "objective/kl": 14.017477035522461, "objective/entropy": -137.78683471679688, "objective/non_score_reward": -0.700873851776123, "objective/rlhf_reward": 8.428031921386719, "objective/scores": 9.125, "policy/approxkl_avg": 0.0017388606211170554, "policy/clipfrac_avg": 0.022260189056396484, "loss/policy_avg": -0.16343963146209717, "loss/value_avg": 0.9192260503768921, "val/clipfrac_avg": 0.23833248019218445, "policy/entropy_avg": 0.17416727542877197, "val/ratio": 0.9880942106246948, "val/ratio_var": 0.00013718391710426658, "val/num_eos_tokens": 155, "lr": 9.92935509259118e-06, "episode": 2016, "epoch": 0.10181818181818182, "step": 126}, {"eps": 0, "objective/kl": 10.212955474853516, "objective/entropy": -77.51470947265625, "objective/non_score_reward": -0.5106477737426758, "objective/rlhf_reward": 8.598727226257324, "objective/scores": 9.125, "policy/approxkl_avg": 0.0012595304287970066, "policy/clipfrac_avg": 0.016321320086717606, "loss/policy_avg": -0.10418622195720673, "loss/value_avg": 2.174149751663208, "val/clipfrac_avg": 0.32252413034439087, "policy/entropy_avg": 0.16196727752685547, "val/ratio": 0.992402195930481, "val/ratio_var": 0.0001610841864021495, "val/num_eos_tokens": 544, "lr": 9.927100106776213e-06, "episode": 2032, "epoch": 0.10262626262626262, "step": 127}, {"eps": 0, "objective/kl": 9.301607131958008, "objective/entropy": -71.08393859863281, "objective/non_score_reward": -0.4650803208351135, "objective/rlhf_reward": 8.554450988769531, "objective/scores": 9.0, "policy/approxkl_avg": 0.0004891916178166866, "policy/clipfrac_avg": 0.006563554517924786, "loss/policy_avg": 0.5139014720916748, "loss/value_avg": 1.2218286991119385, "val/clipfrac_avg": 0.18050439655780792, "policy/entropy_avg": 0.09600615501403809, "val/ratio": 0.9929656386375427, "val/ratio_var": 3.0018733014003374e-05, "val/num_eos_tokens": 269, "lr": 9.924809958745866e-06, "episode": 2048, "epoch": 0.10343434343434343, "step": 128}, {"eps": 0, "objective/kl": 10.98521900177002, "objective/entropy": -131.27288818359375, "objective/non_score_reward": -0.5492609739303589, "objective/rlhf_reward": 9.403863906860352, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0006737498915754259, "policy/clipfrac_avg": 0.010425791144371033, "loss/policy_avg": -0.4225575625896454, "loss/value_avg": 0.8013542294502258, "val/clipfrac_avg": 0.3558182716369629, "policy/entropy_avg": 0.12466853857040405, "val/ratio": 0.9916852712631226, "val/ratio_var": 5.279077231534757e-05, "val/num_eos_tokens": 268, "lr": 9.922484664843763e-06, "episode": 2064, "epoch": 0.10424242424242425, "step": 129}, {"eps": 0, "objective/kl": 10.488237380981445, "objective/entropy": -134.74630737304688, "objective/non_score_reward": -0.5244117975234985, "objective/rlhf_reward": 7.772463798522949, "objective/scores": 8.3125, "policy/approxkl_avg": 0.0009083844488486648, "policy/clipfrac_avg": 0.022758150473237038, "loss/policy_avg": 0.6694027185440063, "loss/value_avg": 2.389725685119629, "val/clipfrac_avg": 0.3100450038909912, "policy/entropy_avg": 0.17615556716918945, "val/ratio": 0.9912290573120117, "val/ratio_var": 4.8142654122784734e-05, "val/num_eos_tokens": 43, "lr": 9.920124241664349e-06, "episode": 2080, "epoch": 0.10505050505050505, "step": 130}, {"eps": 0, "objective/kl": 8.395157814025879, "objective/entropy": -104.74658966064453, "objective/non_score_reward": -0.4197579026222229, "objective/rlhf_reward": 8.465007781982422, "objective/scores": 8.875, "policy/approxkl_avg": 0.0016920489724725485, "policy/clipfrac_avg": 0.02493302896618843, "loss/policy_avg": -0.26812419295310974, "loss/value_avg": 1.8501536846160889, "val/clipfrac_avg": 0.21037200093269348, "policy/entropy_avg": 0.1534414291381836, "val/ratio": 0.990479588508606, "val/ratio_var": 0.00021930081129539758, "val/num_eos_tokens": 339, "lr": 9.917728706052765e-06, "episode": 2096, "epoch": 0.10585858585858586, "step": 131}, {"eps": 0, "objective/kl": 13.195899963378906, "objective/entropy": -64.35730743408203, "objective/non_score_reward": -0.6597949266433716, "objective/rlhf_reward": 8.508173942565918, "objective/scores": 9.1875, "policy/approxkl_avg": 0.0009221321670338511, "policy/clipfrac_avg": 0.004325927700847387, "loss/policy_avg": -0.34324708580970764, "loss/value_avg": 0.6897234916687012, "val/clipfrac_avg": 0.07538644224405289, "policy/entropy_avg": 0.1800684928894043, "val/ratio": 0.9881160259246826, "val/ratio_var": 6.755861249985173e-05, "val/num_eos_tokens": 210, "lr": 9.915298075104735e-06, "episode": 2112, "epoch": 0.10666666666666667, "step": 132}, {"eps": 0, "objective/kl": 10.798791885375977, "objective/entropy": -45.32240295410156, "objective/non_score_reward": -0.5399396419525146, "objective/rlhf_reward": 7.835060119628906, "objective/scores": 8.375, "policy/approxkl_avg": 0.0035242601297795773, "policy/clipfrac_avg": 0.03283273056149483, "loss/policy_avg": -0.04767410457134247, "loss/value_avg": 0.7692283391952515, "val/clipfrac_avg": 0.21977931261062622, "policy/entropy_avg": 0.22318172454833984, "val/ratio": 0.9933129549026489, "val/ratio_var": 0.0021998623851686716, "val/num_eos_tokens": 493, "lr": 9.912832366166443e-06, "episode": 2128, "epoch": 0.10747474747474747, "step": 133}, {"eps": 0, "objective/kl": 12.162965774536133, "objective/entropy": -128.28472900390625, "objective/non_score_reward": -0.6081483364105225, "objective/rlhf_reward": 9.395757675170898, "objective/scores": 10.0, "policy/approxkl_avg": 0.0009850980713963509, "policy/clipfrac_avg": 0.021821077913045883, "loss/policy_avg": -0.5322861075401306, "loss/value_avg": 2.0256404876708984, "val/clipfrac_avg": 0.3014891743659973, "policy/entropy_avg": 0.12530064582824707, "val/ratio": 0.9908243417739868, "val/ratio_var": 0.00012693816097453237, "val/num_eos_tokens": 6, "lr": 9.910331596834399e-06, "episode": 2144, "epoch": 0.10828282828282829, "step": 134}, {"eps": 0, "objective/kl": 12.188176155090332, "objective/entropy": -63.13277053833008, "objective/non_score_reward": -0.6094088554382324, "objective/rlhf_reward": 9.10934066772461, "objective/scores": 9.75, "policy/approxkl_avg": 0.0009403219446539879, "policy/clipfrac_avg": 0.014169535599648952, "loss/policy_avg": -0.3030897378921509, "loss/value_avg": 2.1831560134887695, "val/clipfrac_avg": 0.24678541719913483, "policy/entropy_avg": 0.15417695045471191, "val/ratio": 0.9922115802764893, "val/ratio_var": 0.00014833426394034177, "val/num_eos_tokens": 481, "lr": 9.907795784955327e-06, "episode": 2160, "epoch": 0.10909090909090909, "step": 135}, {"eps": 0, "objective/kl": 10.518301010131836, "objective/entropy": -103.9739990234375, "objective/non_score_reward": -0.5259150862693787, "objective/rlhf_reward": 10.681116104125977, "objective/scores": 11.1875, "policy/approxkl_avg": 0.0007561782840639353, "policy/clipfrac_avg": 0.0030004261061549187, "loss/policy_avg": -0.8595305681228638, "loss/value_avg": 2.5992929935455322, "val/clipfrac_avg": 0.35328760743141174, "policy/entropy_avg": 0.18250799179077148, "val/ratio": 0.9901064038276672, "val/ratio_var": 6.593725993297994e-05, "val/num_eos_tokens": 146, "lr": 9.905224948626033e-06, "episode": 2176, "epoch": 0.1098989898989899, "step": 136}, {"eps": 0, "objective/kl": 14.112113952636719, "objective/entropy": -123.25853729248047, "objective/non_score_reward": -0.705605685710907, "objective/rlhf_reward": 8.704550743103027, "objective/scores": 9.4375, "policy/approxkl_avg": 0.001142181921750307, "policy/clipfrac_avg": 0.025173479691147804, "loss/policy_avg": 0.48323243856430054, "loss/value_avg": 2.3285460472106934, "val/clipfrac_avg": 0.17816151678562164, "policy/entropy_avg": 0.13131451606750488, "val/ratio": 0.9913126230239868, "val/ratio_var": 5.5185322707984596e-05, "val/num_eos_tokens": 190, "lr": 9.90261910619327e-06, "episode": 2192, "epoch": 0.11070707070707071, "step": 137}, {"eps": 0, "objective/kl": 14.988029479980469, "objective/entropy": -83.2667465209961, "objective/non_score_reward": -0.7494014501571655, "objective/rlhf_reward": 8.887317657470703, "objective/scores": 9.625, "policy/approxkl_avg": 0.0032717131543904543, "policy/clipfrac_avg": 0.034809306263923645, "loss/policy_avg": 0.041312795132398605, "loss/value_avg": 1.2238047122955322, "val/clipfrac_avg": 0.12297101318836212, "policy/entropy_avg": 0.19169330596923828, "val/ratio": 0.9875979423522949, "val/ratio_var": 0.00023365950619336218, "val/num_eos_tokens": 458, "lr": 9.899978276253617e-06, "episode": 2208, "epoch": 0.11151515151515151, "step": 138}, {"eps": 0, "objective/kl": 10.463726043701172, "objective/entropy": -151.2768096923828, "objective/non_score_reward": -0.5231863260269165, "objective/rlhf_reward": 9.847908020019531, "objective/scores": 10.375, "policy/approxkl_avg": 0.0015597918536514044, "policy/clipfrac_avg": 0.03354030102491379, "loss/policy_avg": 0.10726683586835861, "loss/value_avg": 1.2004472017288208, "val/clipfrac_avg": 0.25066325068473816, "policy/entropy_avg": 0.13694477081298828, "val/ratio": 0.9888541102409363, "val/ratio_var": 0.0001727374765323475, "val/num_eos_tokens": 461, "lr": 9.897302477653334e-06, "episode": 2224, "epoch": 0.11232323232323232, "step": 139}, {"eps": 0, "objective/kl": 12.376513481140137, "objective/entropy": -56.48613739013672, "objective/non_score_reward": -0.6188256740570068, "objective/rlhf_reward": 7.490549564361572, "objective/scores": 8.125, "policy/approxkl_avg": 0.0009117410518229008, "policy/clipfrac_avg": 0.013704899698495865, "loss/policy_avg": -0.1315029263496399, "loss/value_avg": 1.1189866065979004, "val/clipfrac_avg": 0.12030020356178284, "policy/entropy_avg": 0.14188098907470703, "val/ratio": 0.9916878938674927, "val/ratio_var": 7.079278293531388e-05, "val/num_eos_tokens": 115, "lr": 9.894591729488243e-06, "episode": 2240, "epoch": 0.11313131313131314, "step": 140}, {"eps": 0, "objective/kl": 13.879895210266113, "objective/entropy": -100.32903289794922, "objective/non_score_reward": -0.6939947605133057, "objective/rlhf_reward": 7.934910774230957, "objective/scores": 8.625, "policy/approxkl_avg": 0.0008554073283448815, "policy/clipfrac_avg": 0.01539595890790224, "loss/policy_avg": -0.031752824783325195, "loss/value_avg": 0.9798139333724976, "val/clipfrac_avg": 0.11368511617183685, "policy/entropy_avg": 0.14811944961547852, "val/ratio": 0.9901875257492065, "val/ratio_var": 6.501925963675603e-05, "val/num_eos_tokens": 926, "lr": 9.891846051103578e-06, "episode": 2256, "epoch": 0.11393939393939394, "step": 141}, {"eps": 0, "objective/kl": 14.766319274902344, "objective/entropy": -61.576045989990234, "objective/non_score_reward": -0.7383160591125488, "objective/rlhf_reward": 7.910121440887451, "objective/scores": 8.625, "policy/approxkl_avg": 0.0012796547962352633, "policy/clipfrac_avg": 0.01897471211850643, "loss/policy_avg": 0.16777145862579346, "loss/value_avg": 1.338819980621338, "val/clipfrac_avg": 0.10001248121261597, "policy/entropy_avg": 0.15843772888183594, "val/ratio": 0.9893121123313904, "val/ratio_var": 4.402951162774116e-05, "val/num_eos_tokens": 250, "lr": 9.889065462093851e-06, "episode": 2272, "epoch": 0.11474747474747475, "step": 142}, {"eps": 0, "objective/kl": 16.068700790405273, "objective/entropy": -124.786376953125, "objective/non_score_reward": -0.8034350872039795, "objective/rlhf_reward": 8.337189674377441, "objective/scores": 9.125, "policy/approxkl_avg": 0.0009665879188105464, "policy/clipfrac_avg": 0.02759074792265892, "loss/policy_avg": 0.2867237329483032, "loss/value_avg": 2.6249804496765137, "val/clipfrac_avg": 0.2618255913257599, "policy/entropy_avg": 0.26673293113708496, "val/ratio": 0.9918559789657593, "val/ratio_var": 4.804971831617877e-05, "val/num_eos_tokens": 87, "lr": 9.88624998230272e-06, "episode": 2288, "epoch": 0.11555555555555555, "step": 143}, {"eps": 0, "objective/kl": 14.85019588470459, "objective/entropy": -264.2563171386719, "objective/non_score_reward": -0.7425098419189453, "objective/rlhf_reward": 6.929365158081055, "objective/scores": 7.6875, "policy/approxkl_avg": 0.0007313055102713406, "policy/clipfrac_avg": 0.03529048711061478, "loss/policy_avg": 0.6126020550727844, "loss/value_avg": 1.5799771547317505, "val/clipfrac_avg": 0.16758260130882263, "policy/entropy_avg": 0.1719745397567749, "val/ratio": 0.9931252002716064, "val/ratio_var": 4.071718649356626e-05, "val/num_eos_tokens": 104, "lr": 9.883399631822836e-06, "episode": 2304, "epoch": 0.11636363636363636, "step": 144}, {"eps": 0, "objective/kl": 17.772851943969727, "objective/entropy": -43.337398529052734, "objective/non_score_reward": -0.8886426687240601, "objective/rlhf_reward": 7.296904563903809, "objective/scores": 8.1875, "policy/approxkl_avg": 0.0008921563276089728, "policy/clipfrac_avg": 0.012217779643833637, "loss/policy_avg": 0.11925015598535538, "loss/value_avg": 1.104990005493164, "val/clipfrac_avg": 0.2284112274646759, "policy/entropy_avg": 0.17539596557617188, "val/ratio": 0.9908536672592163, "val/ratio_var": 6.429313361877576e-05, "val/num_eos_tokens": 268, "lr": 9.880514430995703e-06, "episode": 2320, "epoch": 0.11717171717171718, "step": 145}, {"eps": 0, "objective/kl": 13.699361801147461, "objective/entropy": -95.0362777709961, "objective/non_score_reward": -0.684968113899231, "objective/rlhf_reward": 10.244718551635742, "objective/scores": 10.9375, "policy/approxkl_avg": 0.015414729714393616, "policy/clipfrac_avg": 0.041790302842855453, "loss/policy_avg": 0.08043037354946136, "loss/value_avg": 1.5215954780578613, "val/clipfrac_avg": 0.18847554922103882, "policy/entropy_avg": 0.11050856113433838, "val/ratio": 0.990708589553833, "val/ratio_var": 0.00013217887317296118, "val/num_eos_tokens": 473, "lr": 9.87759440041154e-06, "episode": 2336, "epoch": 0.11797979797979798, "step": 146}, {"eps": 0, "objective/kl": 15.634238243103027, "objective/entropy": -81.37150573730469, "objective/non_score_reward": -0.7817119359970093, "objective/rlhf_reward": 10.054224967956543, "objective/scores": 10.8125, "policy/approxkl_avg": 0.001198675250634551, "policy/clipfrac_avg": 0.00917242094874382, "loss/policy_avg": -0.0913308784365654, "loss/value_avg": 1.0237717628479004, "val/clipfrac_avg": 0.20073412358760834, "policy/entropy_avg": 0.14813309907913208, "val/ratio": 0.9932203888893127, "val/ratio_var": 0.00023357552709057927, "val/num_eos_tokens": 175, "lr": 9.874639560909118e-06, "episode": 2352, "epoch": 0.11878787878787879, "step": 147}, {"eps": 0, "objective/kl": 18.754764556884766, "objective/entropy": -58.469085693359375, "objective/non_score_reward": -0.937738299369812, "objective/rlhf_reward": 7.876714706420898, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0012335460633039474, "policy/clipfrac_avg": 0.015984678640961647, "loss/policy_avg": -0.0016996562480926514, "loss/value_avg": 0.8599748611450195, "val/clipfrac_avg": 0.2339939922094345, "policy/entropy_avg": 0.2137432098388672, "val/ratio": 0.9876729249954224, "val/ratio_var": 0.0001175389188574627, "val/num_eos_tokens": 339, "lr": 9.871649933575636e-06, "episode": 2368, "epoch": 0.1195959595959596, "step": 148}, {"eps": 0, "objective/kl": 17.043121337890625, "objective/entropy": -153.42767333984375, "objective/non_score_reward": -0.8521560430526733, "objective/rlhf_reward": 7.327531337738037, "objective/scores": 8.1875, "policy/approxkl_avg": 0.0007452386198565364, "policy/clipfrac_avg": 0.004154819063842297, "loss/policy_avg": -0.3132669925689697, "loss/value_avg": 0.7471304535865784, "val/clipfrac_avg": 0.27733689546585083, "policy/entropy_avg": 0.13955670595169067, "val/ratio": 0.9911587238311768, "val/ratio_var": 4.037221151520498e-05, "val/num_eos_tokens": 543, "lr": 9.868625539746545e-06, "episode": 2384, "epoch": 0.1204040404040404, "step": 149}, {"eps": 0, "objective/kl": 15.889145851135254, "objective/entropy": -134.21249389648438, "objective/non_score_reward": -0.7944573163986206, "objective/rlhf_reward": 8.15476131439209, "objective/scores": 8.9375, "policy/approxkl_avg": 0.0008621187880635262, "policy/clipfrac_avg": 0.02879197522997856, "loss/policy_avg": 0.5823547840118408, "loss/value_avg": 2.283393144607544, "val/clipfrac_avg": 0.3507724404335022, "policy/entropy_avg": 0.16280102729797363, "val/ratio": 0.9914773106575012, "val/ratio_var": 6.432795635191724e-05, "val/num_eos_tokens": 742, "lr": 9.86556640100541e-06, "episode": 2400, "epoch": 0.12121212121212122, "step": 150}, {"eps": 0, "objective/kl": 16.739295959472656, "objective/entropy": -73.51275634765625, "objective/non_score_reward": -0.8369647264480591, "objective/rlhf_reward": 10.03412914276123, "objective/scores": 10.875, "policy/approxkl_avg": 0.0008029645541682839, "policy/clipfrac_avg": 0.007894085720181465, "loss/policy_avg": 0.00687573105096817, "loss/value_avg": 0.5364193916320801, "val/clipfrac_avg": 0.25386878848075867, "policy/entropy_avg": 0.15464520454406738, "val/ratio": 0.9904509782791138, "val/ratio_var": 2.9840115530532785e-05, "val/num_eos_tokens": 33, "lr": 9.862472539183757e-06, "episode": 2416, "epoch": 0.12202020202020201, "step": 151}, {"eps": 0, "objective/kl": 14.286821365356445, "objective/entropy": -136.72979736328125, "objective/non_score_reward": -0.7143411040306091, "objective/rlhf_reward": 9.172377586364746, "objective/scores": 9.875, "policy/approxkl_avg": 0.0011772194411605597, "policy/clipfrac_avg": 0.016035620123147964, "loss/policy_avg": 0.46432220935821533, "loss/value_avg": 2.502655267715454, "val/clipfrac_avg": 0.286276638507843, "policy/entropy_avg": 0.1345546841621399, "val/ratio": 0.9900628328323364, "val/ratio_var": 7.126816490199417e-05, "val/num_eos_tokens": 619, "lr": 9.859343976360904e-06, "episode": 2432, "epoch": 0.12282828282828283, "step": 152}, {"eps": 0, "objective/kl": 16.09568214416504, "objective/entropy": -138.69039916992188, "objective/non_score_reward": -0.8047841191291809, "objective/rlhf_reward": 8.843653678894043, "objective/scores": 9.625, "policy/approxkl_avg": 0.0012001043651252985, "policy/clipfrac_avg": 0.0180644653737545, "loss/policy_avg": 0.09378673136234283, "loss/value_avg": 3.2477402687072754, "val/clipfrac_avg": 0.2985517382621765, "policy/entropy_avg": 0.149375319480896, "val/ratio": 0.9893068671226501, "val/ratio_var": 6.621519423788413e-05, "val/num_eos_tokens": 433, "lr": 9.85618073486382e-06, "episode": 2448, "epoch": 0.12363636363636364, "step": 153}, {"eps": 0, "objective/kl": 19.26645278930664, "objective/entropy": -80.84510803222656, "objective/non_score_reward": -0.963322639465332, "objective/rlhf_reward": 8.341363906860352, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0007789209485054016, "policy/clipfrac_avg": 0.006222645752131939, "loss/policy_avg": -0.3594944477081299, "loss/value_avg": 2.61572527885437, "val/clipfrac_avg": 0.14149022102355957, "policy/entropy_avg": 0.12923431396484375, "val/ratio": 0.9912763237953186, "val/ratio_var": 5.006176434108056e-05, "val/num_eos_tokens": 65, "lr": 9.852982837266955e-06, "episode": 2464, "epoch": 0.12444444444444444, "step": 154}, {"eps": 0, "objective/kl": 16.327844619750977, "objective/entropy": -114.74142456054688, "objective/non_score_reward": -0.8163923025131226, "objective/rlhf_reward": 9.566420555114746, "objective/scores": 10.375, "policy/approxkl_avg": 0.001046534744091332, "policy/clipfrac_avg": 0.01326068863272667, "loss/policy_avg": 0.13503910601139069, "loss/value_avg": 1.3053985834121704, "val/clipfrac_avg": 0.12556566298007965, "policy/entropy_avg": 0.13961029052734375, "val/ratio": 0.9914346933364868, "val/ratio_var": 8.751903078518808e-05, "val/num_eos_tokens": 24, "lr": 9.849750306392085e-06, "episode": 2480, "epoch": 0.12525252525252525, "step": 155}, {"eps": 0, "objective/kl": 14.01751708984375, "objective/entropy": -51.69160461425781, "objective/non_score_reward": -0.7008758783340454, "objective/rlhf_reward": 9.084280014038086, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0020042983815073967, "policy/clipfrac_avg": 0.025917891412973404, "loss/policy_avg": -0.2862533926963806, "loss/value_avg": 1.4972138404846191, "val/clipfrac_avg": 0.07307614386081696, "policy/entropy_avg": 0.18289995193481445, "val/ratio": 0.988426923751831, "val/ratio_var": 0.00010893259604927152, "val/num_eos_tokens": 89, "lr": 9.846483165308142e-06, "episode": 2496, "epoch": 0.12606060606060607, "step": 156}, {"eps": 0, "objective/kl": 18.988895416259766, "objective/entropy": -38.66559600830078, "objective/non_score_reward": -0.9494448304176331, "objective/rlhf_reward": 8.070086479187012, "objective/scores": 9.0, "policy/approxkl_avg": 0.004971734713762999, "policy/clipfrac_avg": 0.017674520611763, "loss/policy_avg": 0.08509957790374756, "loss/value_avg": 1.0140475034713745, "val/clipfrac_avg": 0.1631356030702591, "policy/entropy_avg": 0.19965362548828125, "val/ratio": 0.9906042218208313, "val/ratio_var": 0.00015353818889707327, "val/num_eos_tokens": 85, "lr": 9.843181437331055e-06, "episode": 2512, "epoch": 0.12686868686868688, "step": 157}, {"eps": 0, "objective/kl": 14.4805908203125, "objective/entropy": -81.05842590332031, "objective/non_score_reward": -0.724029541015625, "objective/rlhf_reward": 9.068939208984375, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0011242629261687398, "policy/clipfrac_avg": 0.03127668797969818, "loss/policy_avg": -0.30014684796333313, "loss/value_avg": 2.4983510971069336, "val/clipfrac_avg": 0.15103457868099213, "policy/entropy_avg": 0.18725228309631348, "val/ratio": 0.9933140277862549, "val/ratio_var": 0.00010324850882170722, "val/num_eos_tokens": 3, "lr": 9.839845146023583e-06, "episode": 2528, "epoch": 0.12767676767676767, "step": 158}, {"eps": 0, "objective/kl": 18.89221954345703, "objective/entropy": -75.23670196533203, "objective/non_score_reward": -0.9446109533309937, "objective/rlhf_reward": 9.160857200622559, "objective/scores": 10.125, "policy/approxkl_avg": 0.005313557107001543, "policy/clipfrac_avg": 0.017709676176309586, "loss/policy_avg": 0.03238319233059883, "loss/value_avg": 0.9245697855949402, "val/clipfrac_avg": 0.09926828742027283, "policy/entropy_avg": 0.2630615234375, "val/ratio": 0.9845870137214661, "val/ratio_var": 0.0004384346539154649, "val/num_eos_tokens": 214, "lr": 9.836474315195148e-06, "episode": 2544, "epoch": 0.12848484848484848, "step": 159}, {"eps": 0, "objective/kl": 14.313985824584961, "objective/entropy": -58.01062774658203, "objective/non_score_reward": -0.7156991958618164, "objective/rlhf_reward": 7.667113304138184, "objective/scores": 8.375, "policy/approxkl_avg": 0.0008890245808288455, "policy/clipfrac_avg": 0.00987298134714365, "loss/policy_avg": -0.00330284982919693, "loss/value_avg": 1.9382867813110352, "val/clipfrac_avg": 0.11537374556064606, "policy/entropy_avg": 0.19803333282470703, "val/ratio": 0.9886167049407959, "val/ratio_var": 5.6686280004214495e-05, "val/num_eos_tokens": 85, "lr": 9.833068968901657e-06, "episode": 2560, "epoch": 0.1292929292929293, "step": 160}, {"eps": 0, "objective/kl": 14.487106323242188, "objective/entropy": -105.31230163574219, "objective/non_score_reward": -0.7243553400039673, "objective/rlhf_reward": 9.494394302368164, "objective/scores": 10.25, "policy/approxkl_avg": 0.0008457785006612539, "policy/clipfrac_avg": 0.01930004358291626, "loss/policy_avg": 0.15009576082229614, "loss/value_avg": 1.3036422729492188, "val/clipfrac_avg": 0.20280539989471436, "policy/entropy_avg": 0.16261720657348633, "val/ratio": 0.9908741116523743, "val/ratio_var": 8.265120413852856e-05, "val/num_eos_tokens": 333, "lr": 9.829629131445342e-06, "episode": 2576, "epoch": 0.1301010101010101, "step": 161}, {"eps": 0, "objective/kl": 16.28435516357422, "objective/entropy": -119.47211456298828, "objective/non_score_reward": -0.8142178058624268, "objective/rlhf_reward": 6.927969455718994, "objective/scores": 7.75, "policy/approxkl_avg": 0.0012012721272185445, "policy/clipfrac_avg": 0.03371775150299072, "loss/policy_avg": 0.7536355257034302, "loss/value_avg": 2.164299249649048, "val/clipfrac_avg": 0.365829735994339, "policy/entropy_avg": 0.2003459930419922, "val/ratio": 0.9901646971702576, "val/ratio_var": 9.40158570301719e-05, "val/num_eos_tokens": 737, "lr": 9.826154827374578e-06, "episode": 2592, "epoch": 0.13090909090909092, "step": 162}, {"eps": 0, "objective/kl": 17.124767303466797, "objective/entropy": -88.83195495605469, "objective/non_score_reward": -0.8562383651733398, "objective/rlhf_reward": 9.327356338500977, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0007394744316115975, "policy/clipfrac_avg": 0.0198641587048769, "loss/policy_avg": -0.27083319425582886, "loss/value_avg": 1.6828413009643555, "val/clipfrac_avg": 0.3026716709136963, "policy/entropy_avg": 0.13005447387695312, "val/ratio": 0.9908427000045776, "val/ratio_var": 6.518982991110533e-05, "val/num_eos_tokens": 512, "lr": 9.822646081483713e-06, "episode": 2608, "epoch": 0.1317171717171717, "step": 163}, {"eps": 0, "objective/kl": 15.069713592529297, "objective/entropy": -210.48764038085938, "objective/non_score_reward": -0.7534856796264648, "objective/rlhf_reward": 7.668389320373535, "objective/scores": 8.4375, "policy/approxkl_avg": 0.000925661763176322, "policy/clipfrac_avg": 0.02412327565252781, "loss/policy_avg": -0.37486350536346436, "loss/value_avg": 2.394012689590454, "val/clipfrac_avg": 0.14958947896957397, "policy/entropy_avg": 0.16263806819915771, "val/ratio": 0.9890938997268677, "val/ratio_var": 0.0001162591579486616, "val/num_eos_tokens": 676, "lr": 9.819102918812887e-06, "episode": 2624, "epoch": 0.13252525252525252, "step": 164}, {"eps": 0, "objective/kl": 16.533164978027344, "objective/entropy": -53.14797592163086, "objective/non_score_reward": -0.8266582489013672, "objective/rlhf_reward": 8.817873001098633, "objective/scores": 9.625, "policy/approxkl_avg": 0.0009110436076298356, "policy/clipfrac_avg": 0.005712536629289389, "loss/policy_avg": -0.3021676242351532, "loss/value_avg": 0.7970246076583862, "val/clipfrac_avg": 0.15537136793136597, "policy/entropy_avg": 0.17499542236328125, "val/ratio": 0.9888813495635986, "val/ratio_var": 3.097565422649495e-05, "val/num_eos_tokens": 536, "lr": 9.815525364647853e-06, "episode": 2640, "epoch": 0.13333333333333333, "step": 165}, {"eps": 0, "objective/kl": 12.42782974243164, "objective/entropy": -61.10470962524414, "objective/non_score_reward": -0.6213915348052979, "objective/rlhf_reward": 8.554389953613281, "objective/scores": 9.1875, "policy/approxkl_avg": 0.0008850512094795704, "policy/clipfrac_avg": 0.013360471464693546, "loss/policy_avg": 0.10213343799114227, "loss/value_avg": 0.985383927822113, "val/clipfrac_avg": 0.25333312153816223, "policy/entropy_avg": 0.1629645824432373, "val/ratio": 0.9904332160949707, "val/ratio_var": 7.648977043572813e-05, "val/num_eos_tokens": 73, "lr": 9.811913444519803e-06, "episode": 2656, "epoch": 0.13414141414141414, "step": 166}, {"eps": 0, "objective/kl": 14.755228996276855, "objective/entropy": -38.80767059326172, "objective/non_score_reward": -0.7377614974975586, "objective/rlhf_reward": 7.594269752502441, "objective/scores": 8.3125, "policy/approxkl_avg": 0.0033133821561932564, "policy/clipfrac_avg": 0.012325381860136986, "loss/policy_avg": 0.5110056400299072, "loss/value_avg": 1.0773067474365234, "val/clipfrac_avg": 0.166399747133255, "policy/entropy_avg": 0.20802688598632812, "val/ratio": 0.9878016710281372, "val/ratio_var": 5.6116346968337893e-05, "val/num_eos_tokens": 280, "lr": 9.808267184205182e-06, "episode": 2672, "epoch": 0.13494949494949496, "step": 167}, {"eps": 0, "objective/kl": 14.000960350036621, "objective/entropy": -45.07624816894531, "objective/non_score_reward": -0.70004802942276, "objective/rlhf_reward": 9.370265007019043, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0028054637368768454, "policy/clipfrac_avg": 0.017689436674118042, "loss/policy_avg": -0.0654146745800972, "loss/value_avg": 1.8866548538208008, "val/clipfrac_avg": 0.4012067914009094, "policy/entropy_avg": 0.20137405395507812, "val/ratio": 0.984307050704956, "val/ratio_var": 0.0002625912893563509, "val/num_eos_tokens": 219, "lr": 9.804586609725499e-06, "episode": 2688, "epoch": 0.13575757575757577, "step": 168}, {"eps": 0, "objective/kl": 12.764520645141602, "objective/entropy": -29.569345474243164, "objective/non_score_reward": -0.6382260322570801, "objective/rlhf_reward": 9.596149444580078, "objective/scores": 10.25, "policy/approxkl_avg": 0.0026277583092451096, "policy/clipfrac_avg": 0.020652472972869873, "loss/policy_avg": -0.26235154271125793, "loss/value_avg": 0.9361153841018677, "val/clipfrac_avg": 0.07669641077518463, "policy/entropy_avg": 0.1861964464187622, "val/ratio": 0.9880188703536987, "val/ratio_var": 0.0003732522891368717, "val/num_eos_tokens": 114, "lr": 9.800871747347148e-06, "episode": 2704, "epoch": 0.13656565656565656, "step": 169}, {"eps": 0, "objective/kl": 14.564604759216309, "objective/entropy": -115.31767272949219, "objective/non_score_reward": -0.7282302379608154, "objective/rlhf_reward": 8.541301727294922, "objective/scores": 9.25, "policy/approxkl_avg": 0.0009797554230317473, "policy/clipfrac_avg": 0.013546416535973549, "loss/policy_avg": 0.0036516934633255005, "loss/value_avg": 0.7808534502983093, "val/clipfrac_avg": 0.13324236869812012, "policy/entropy_avg": 0.17569169402122498, "val/ratio": 0.9880359172821045, "val/ratio_var": 5.694741048500873e-05, "val/num_eos_tokens": 24, "lr": 9.797122623581217e-06, "episode": 2720, "epoch": 0.13737373737373737, "step": 170}, {"eps": 0, "objective/kl": 17.099538803100586, "objective/entropy": -65.40606689453125, "objective/non_score_reward": -0.854977011680603, "objective/rlhf_reward": 8.246585845947266, "objective/scores": 9.125, "policy/approxkl_avg": 0.0009173300350084901, "policy/clipfrac_avg": 0.011180122382938862, "loss/policy_avg": -0.3128410577774048, "loss/value_avg": 0.6949845552444458, "val/clipfrac_avg": 0.09039227664470673, "policy/entropy_avg": 0.17095184326171875, "val/ratio": 0.9889956116676331, "val/ratio_var": 5.041521944804117e-05, "val/num_eos_tokens": 391, "lr": 9.793339265183303e-06, "episode": 2736, "epoch": 0.13818181818181818, "step": 171}, {"eps": 0, "objective/kl": 14.352700233459473, "objective/entropy": -149.44508361816406, "objective/non_score_reward": -0.7176350355148315, "objective/rlhf_reward": 7.446427345275879, "objective/scores": 8.1875, "policy/approxkl_avg": 0.000751341343857348, "policy/clipfrac_avg": 0.014955854043364525, "loss/policy_avg": -0.06095901131629944, "loss/value_avg": 0.6817218065261841, "val/clipfrac_avg": 0.3020364046096802, "policy/entropy_avg": 0.1538681983947754, "val/ratio": 0.9901411533355713, "val/ratio_var": 5.242693077889271e-05, "val/num_eos_tokens": 70, "lr": 9.78952169915332e-06, "episode": 2752, "epoch": 0.138989898989899, "step": 172}, {"eps": 0, "objective/kl": 18.063003540039062, "objective/entropy": -103.2147216796875, "objective/non_score_reward": -0.9031502604484558, "objective/rlhf_reward": 10.32341194152832, "objective/scores": 11.25, "policy/approxkl_avg": 0.0011648546205833554, "policy/clipfrac_avg": 0.0217579435557127, "loss/policy_avg": -0.9073721170425415, "loss/value_avg": 3.3096258640289307, "val/clipfrac_avg": 0.36076119542121887, "policy/entropy_avg": 0.18919309973716736, "val/ratio": 0.9889928102493286, "val/ratio_var": 0.00014562088472303003, "val/num_eos_tokens": 583, "lr": 9.785669952735294e-06, "episode": 2768, "epoch": 0.1397979797979798, "step": 173}, {"eps": 0, "objective/kl": 15.56573486328125, "objective/entropy": -144.74966430664062, "objective/non_score_reward": -0.7782866954803467, "objective/rlhf_reward": 7.422885417938232, "objective/scores": 8.1875, "policy/approxkl_avg": 0.0008035623468458652, "policy/clipfrac_avg": 0.03028782643377781, "loss/policy_avg": 0.9771312475204468, "loss/value_avg": 3.1401190757751465, "val/clipfrac_avg": 0.326850026845932, "policy/entropy_avg": 0.13976645469665527, "val/ratio": 0.9907339215278625, "val/ratio_var": 4.144087870372459e-05, "val/num_eos_tokens": 455, "lr": 9.781784053417192e-06, "episode": 2784, "epoch": 0.1406060606060606, "step": 174}, {"eps": 0, "objective/kl": 14.400297164916992, "objective/entropy": -106.90374755859375, "objective/non_score_reward": -0.7200148701667786, "objective/rlhf_reward": 7.998735427856445, "objective/scores": 8.75, "policy/approxkl_avg": 0.0009868306806311011, "policy/clipfrac_avg": 0.009237098507583141, "loss/policy_avg": 0.28167733550071716, "loss/value_avg": 1.6407313346862793, "val/clipfrac_avg": 0.19442003965377808, "policy/entropy_avg": 0.16825366020202637, "val/ratio": 0.9899091720581055, "val/ratio_var": 9.541668259771541e-05, "val/num_eos_tokens": 59, "lr": 9.777864028930705e-06, "episode": 2800, "epoch": 0.1414141414141414, "step": 175}, {"eps": 0, "objective/kl": 9.653593063354492, "objective/entropy": -60.392662048339844, "objective/non_score_reward": -0.48267969489097595, "objective/rlhf_reward": 9.052475929260254, "objective/scores": 9.5625, "policy/approxkl_avg": 0.0010749915381893516, "policy/clipfrac_avg": 0.00982856098562479, "loss/policy_avg": -0.15210214257240295, "loss/value_avg": 1.2273833751678467, "val/clipfrac_avg": 0.11848568916320801, "policy/entropy_avg": 0.16010570526123047, "val/ratio": 0.9885416030883789, "val/ratio_var": 5.872864130651578e-05, "val/num_eos_tokens": 58, "lr": 9.77390990725106e-06, "episode": 2816, "epoch": 0.14222222222222222, "step": 176}, {"eps": 0, "objective/kl": 11.461904525756836, "objective/entropy": -69.82381439208984, "objective/non_score_reward": -0.5730952620506287, "objective/rlhf_reward": 11.044092178344727, "objective/scores": 11.625, "policy/approxkl_avg": 0.0013258052058517933, "policy/clipfrac_avg": 0.02767137438058853, "loss/policy_avg": 0.17394450306892395, "loss/value_avg": 3.3872365951538086, "val/clipfrac_avg": 0.25187912583351135, "policy/entropy_avg": 0.15888667106628418, "val/ratio": 0.9916998744010925, "val/ratio_var": 7.523155363742262e-05, "val/num_eos_tokens": 28, "lr": 9.76992171659682e-06, "episode": 2832, "epoch": 0.14303030303030304, "step": 177}, {"eps": 0, "objective/kl": 16.81972312927246, "objective/entropy": -124.40007019042969, "objective/non_score_reward": -0.8409861326217651, "objective/rlhf_reward": 10.639482498168945, "objective/scores": 11.5, "policy/approxkl_avg": 0.0010132414754480124, "policy/clipfrac_avg": 0.008814403787255287, "loss/policy_avg": -0.12190955877304077, "loss/value_avg": 1.3829247951507568, "val/clipfrac_avg": 0.2293124943971634, "policy/entropy_avg": 0.15328598022460938, "val/ratio": 0.9888865947723389, "val/ratio_var": 7.727585034444928e-05, "val/num_eos_tokens": 372, "lr": 9.76589948542968e-06, "episode": 2848, "epoch": 0.14383838383838385, "step": 178}, {"eps": 0, "objective/kl": 12.127764701843262, "objective/entropy": -97.340576171875, "objective/non_score_reward": -0.6063882112503052, "objective/rlhf_reward": 8.141658782958984, "objective/scores": 8.75, "policy/approxkl_avg": 0.0008368783164769411, "policy/clipfrac_avg": 0.012604119256138802, "loss/policy_avg": -0.36637282371520996, "loss/value_avg": 1.258082628250122, "val/clipfrac_avg": 0.0866643637418747, "policy/entropy_avg": 0.15296649932861328, "val/ratio": 0.9895975589752197, "val/ratio_var": 6.253716128412634e-05, "val/num_eos_tokens": 53, "lr": 9.761843242454261e-06, "episode": 2864, "epoch": 0.14464646464646463, "step": 179}, {"eps": 0, "objective/kl": 12.383216857910156, "objective/entropy": -92.20721435546875, "objective/non_score_reward": -0.6191608309745789, "objective/rlhf_reward": 10.091776847839355, "objective/scores": 10.6875, "policy/approxkl_avg": 0.0010203049750998616, "policy/clipfrac_avg": 0.020494675263762474, "loss/policy_avg": -0.7018676996231079, "loss/value_avg": 2.839538097381592, "val/clipfrac_avg": 0.19666892290115356, "policy/entropy_avg": 0.14533498883247375, "val/ratio": 0.9898504018783569, "val/ratio_var": 8.984190208138898e-05, "val/num_eos_tokens": 4, "lr": 9.757753016617917e-06, "episode": 2880, "epoch": 0.14545454545454545, "step": 180}, {"eps": 0, "objective/kl": 12.262140274047852, "objective/entropy": -76.44052124023438, "objective/non_score_reward": -0.6131070256233215, "objective/rlhf_reward": 9.078298568725586, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0010697869583964348, "policy/clipfrac_avg": 0.022361809387803078, "loss/policy_avg": -0.12757132947444916, "loss/value_avg": 2.167374849319458, "val/clipfrac_avg": 0.31748759746551514, "policy/entropy_avg": 0.15213966369628906, "val/ratio": 0.9903136491775513, "val/ratio_var": 8.450200402876362e-05, "val/num_eos_tokens": 158, "lr": 9.753628837110514e-06, "episode": 2896, "epoch": 0.14626262626262626, "step": 181}, {"eps": 0, "objective/kl": 12.123230934143066, "objective/entropy": -149.43963623046875, "objective/non_score_reward": -0.6061615943908691, "objective/rlhf_reward": 10.073525428771973, "objective/scores": 10.6875, "policy/approxkl_avg": 0.0008650605450384319, "policy/clipfrac_avg": 0.029723089188337326, "loss/policy_avg": -0.030544668436050415, "loss/value_avg": 1.8187932968139648, "val/clipfrac_avg": 0.1266334354877472, "policy/entropy_avg": 0.14513269066810608, "val/ratio": 0.9940528869628906, "val/ratio_var": 9.982670599129051e-05, "val/num_eos_tokens": 192, "lr": 9.749470733364231e-06, "episode": 2912, "epoch": 0.14707070707070707, "step": 182}, {"eps": 0, "objective/kl": 15.383115768432617, "objective/entropy": -50.62373352050781, "objective/non_score_reward": -0.769155740737915, "objective/rlhf_reward": 9.641000747680664, "objective/scores": 10.4375, "policy/approxkl_avg": 0.004477909300476313, "policy/clipfrac_avg": 0.050629667937755585, "loss/policy_avg": 0.08868739008903503, "loss/value_avg": 1.8785271644592285, "val/clipfrac_avg": 0.2094041109085083, "policy/entropy_avg": 0.2141427993774414, "val/ratio": 0.9843396544456482, "val/ratio_var": 0.00037895137211307883, "val/num_eos_tokens": 2, "lr": 9.745278735053345e-06, "episode": 2928, "epoch": 0.1478787878787879, "step": 183}, {"eps": 0, "objective/kl": 15.159713745117188, "objective/entropy": -82.41788482666016, "objective/non_score_reward": -0.7579857110977173, "objective/rlhf_reward": 7.968576431274414, "objective/scores": 8.75, "policy/approxkl_avg": 0.0010779302101582289, "policy/clipfrac_avg": 0.011320716701447964, "loss/policy_avg": -0.3627840578556061, "loss/value_avg": 0.6112059354782104, "val/clipfrac_avg": 0.15411992371082306, "policy/entropy_avg": 0.17828106880187988, "val/ratio": 0.9890798330307007, "val/ratio_var": 9.126321674557403e-05, "val/num_eos_tokens": 67, "lr": 9.74105287209402e-06, "episode": 2944, "epoch": 0.1486868686868687, "step": 184}, {"eps": 0, "objective/kl": 12.352827072143555, "objective/entropy": -101.67889404296875, "objective/non_score_reward": -0.6176413297653198, "objective/rlhf_reward": 9.17923355102539, "objective/scores": 9.8125, "policy/approxkl_avg": 0.011117171496152878, "policy/clipfrac_avg": 0.017320796847343445, "loss/policy_avg": -0.18067747354507446, "loss/value_avg": 0.5258877277374268, "val/clipfrac_avg": 0.19647085666656494, "policy/entropy_avg": 0.15565109252929688, "val/ratio": 0.984650731086731, "val/ratio_var": 0.0008224447374232113, "val/num_eos_tokens": 210, "lr": 9.736793174644106e-06, "episode": 2960, "epoch": 0.1494949494949495, "step": 185}, {"eps": 0, "objective/kl": 11.292556762695312, "objective/entropy": -65.328369140625, "objective/non_score_reward": -0.5646278262138367, "objective/rlhf_reward": 9.228340148925781, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0006467722705565393, "policy/clipfrac_avg": 0.021413927897810936, "loss/policy_avg": 0.1885536015033722, "loss/value_avg": 1.6109416484832764, "val/clipfrac_avg": 0.18745771050453186, "policy/entropy_avg": 0.11110901832580566, "val/ratio": 0.993016242980957, "val/ratio_var": 3.980214387411252e-05, "val/num_eos_tokens": 246, "lr": 9.732499673102895e-06, "episode": 2976, "epoch": 0.1503030303030303, "step": 186}, {"eps": 0, "objective/kl": 16.007747650146484, "objective/entropy": -124.52982330322266, "objective/non_score_reward": -0.8003873825073242, "objective/rlhf_reward": 7.242581844329834, "objective/scores": 8.0625, "policy/approxkl_avg": 0.0017935727955773473, "policy/clipfrac_avg": 0.021759318187832832, "loss/policy_avg": 0.32616978883743286, "loss/value_avg": 0.6681559681892395, "val/clipfrac_avg": 0.12101784348487854, "policy/entropy_avg": 0.15468502044677734, "val/ratio": 0.989864706993103, "val/ratio_var": 5.842960308655165e-05, "val/num_eos_tokens": 21, "lr": 9.728172398110935e-06, "episode": 2992, "epoch": 0.1511111111111111, "step": 187}, {"eps": 0, "objective/kl": 14.366378784179688, "objective/entropy": -38.704368591308594, "objective/non_score_reward": -0.7183189988136292, "objective/rlhf_reward": 7.746524810791016, "objective/scores": 8.4375, "policy/approxkl_avg": 0.0014909141464158893, "policy/clipfrac_avg": 0.025371305644512177, "loss/policy_avg": 0.1749454289674759, "loss/value_avg": 1.0790375471115112, "val/clipfrac_avg": 0.1147574707865715, "policy/entropy_avg": 0.15520191192626953, "val/ratio": 0.989890456199646, "val/ratio_var": 7.073572487570345e-05, "val/num_eos_tokens": 135, "lr": 9.723811380549792e-06, "episode": 3008, "epoch": 0.15191919191919193, "step": 188}, {"eps": 0, "objective/kl": 13.329991340637207, "objective/entropy": -104.98526000976562, "objective/non_score_reward": -0.6664996147155762, "objective/rlhf_reward": 7.052250385284424, "objective/scores": 7.71875, "policy/approxkl_avg": 0.0009702666429802775, "policy/clipfrac_avg": 0.03182617574930191, "loss/policy_avg": -0.5833231210708618, "loss/value_avg": 3.6844418048858643, "val/clipfrac_avg": 0.15363654494285583, "policy/entropy_avg": 0.1709080934524536, "val/ratio": 0.9898666143417358, "val/ratio_var": 0.00010601755639072508, "val/num_eos_tokens": 1103, "lr": 9.719416651541839e-06, "episode": 3024, "epoch": 0.15272727272727274, "step": 189}, {"eps": 0, "objective/kl": 11.613686561584473, "objective/entropy": -87.40807342529297, "objective/non_score_reward": -0.5806843042373657, "objective/rlhf_reward": 10.415409088134766, "objective/scores": 11.0, "policy/approxkl_avg": 0.0009950872045010328, "policy/clipfrac_avg": 0.010731380432844162, "loss/policy_avg": -0.14998452365398407, "loss/value_avg": 2.081155300140381, "val/clipfrac_avg": 0.20180140435695648, "policy/entropy_avg": 0.1969156265258789, "val/ratio": 0.9887533783912659, "val/ratio_var": 6.801114068366587e-05, "val/num_eos_tokens": 140, "lr": 9.714988242450028e-06, "episode": 3040, "epoch": 0.15353535353535352, "step": 190}, {"eps": 0, "objective/kl": 11.947388648986816, "objective/entropy": -76.69435119628906, "objective/non_score_reward": -0.5973694324493408, "objective/rlhf_reward": 9.894817352294922, "objective/scores": 10.5, "policy/approxkl_avg": 0.0009129929821938276, "policy/clipfrac_avg": 0.006569317542016506, "loss/policy_avg": -0.24951934814453125, "loss/value_avg": 0.500019907951355, "val/clipfrac_avg": 0.13341541588306427, "policy/entropy_avg": 0.16202569007873535, "val/ratio": 0.9887447357177734, "val/ratio_var": 5.932084968662821e-05, "val/num_eos_tokens": 435, "lr": 9.710526184877667e-06, "episode": 3056, "epoch": 0.15434343434343434, "step": 191}, {"eps": 0, "objective/kl": 13.22661304473877, "objective/entropy": -67.43034362792969, "objective/non_score_reward": -0.6613306403160095, "objective/rlhf_reward": 10.838668823242188, "objective/scores": 11.5, "policy/approxkl_avg": 0.0012521710013970733, "policy/clipfrac_avg": 0.011959927156567574, "loss/policy_avg": 0.07577252388000488, "loss/value_avg": 0.5789152979850769, "val/clipfrac_avg": 0.1288057565689087, "policy/entropy_avg": 0.16521143913269043, "val/ratio": 0.9881963133811951, "val/ratio_var": 7.64970391173847e-05, "val/num_eos_tokens": 212, "lr": 9.706030510668202e-06, "episode": 3072, "epoch": 0.15515151515151515, "step": 192}, {"eps": 0, "objective/kl": 10.532510757446289, "objective/entropy": -95.4016342163086, "objective/non_score_reward": -0.5266256332397461, "objective/rlhf_reward": 11.563218116760254, "objective/scores": 12.0625, "policy/approxkl_avg": 0.0009733056649565697, "policy/clipfrac_avg": 0.015730470418930054, "loss/policy_avg": -0.4162842333316803, "loss/value_avg": 2.1575677394866943, "val/clipfrac_avg": 0.2311292290687561, "policy/entropy_avg": 0.16752052307128906, "val/ratio": 0.9886439442634583, "val/ratio_var": 0.00013025107909925282, "val/num_eos_tokens": 344, "lr": 9.70150125190498e-06, "episode": 3088, "epoch": 0.15595959595959596, "step": 193}, {"eps": 0, "objective/kl": 11.387260437011719, "objective/entropy": -155.12109375, "objective/non_score_reward": -0.569362998008728, "objective/rlhf_reward": 8.19040298461914, "objective/scores": 8.75, "policy/approxkl_avg": 0.0007924421224743128, "policy/clipfrac_avg": 0.014981664717197418, "loss/policy_avg": 0.6562298536300659, "loss/value_avg": 1.8542827367782593, "val/clipfrac_avg": 0.21850237250328064, "policy/entropy_avg": 0.15529108047485352, "val/ratio": 0.9898360967636108, "val/ratio_var": 0.000115352297143545, "val/num_eos_tokens": 38, "lr": 9.696938440911025e-06, "episode": 3104, "epoch": 0.15676767676767678, "step": 194}, {"eps": 0, "objective/kl": 14.468893051147461, "objective/entropy": -108.78264617919922, "objective/non_score_reward": -0.7234447002410889, "objective/rlhf_reward": 10.987492561340332, "objective/scores": 11.6875, "policy/approxkl_avg": 0.0011592827504500747, "policy/clipfrac_avg": 0.012323892675340176, "loss/policy_avg": -0.23366859555244446, "loss/value_avg": 1.7562166452407837, "val/clipfrac_avg": 0.3065049946308136, "policy/entropy_avg": 0.13213586807250977, "val/ratio": 0.9917997121810913, "val/ratio_var": 5.691219848813489e-05, "val/num_eos_tokens": 191, "lr": 9.692342110248802e-06, "episode": 3120, "epoch": 0.15757575757575756, "step": 195}, {"eps": 0, "objective/kl": 13.238128662109375, "objective/entropy": -72.25318908691406, "objective/non_score_reward": -0.6619064807891846, "objective/rlhf_reward": 8.775592803955078, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0008679552120156586, "policy/clipfrac_avg": 0.010309899225831032, "loss/policy_avg": -0.12489986419677734, "loss/value_avg": 0.3401644229888916, "val/clipfrac_avg": 0.11943725496530533, "policy/entropy_avg": 0.16907691955566406, "val/ratio": 0.9904986619949341, "val/ratio_var": 6.888979987706989e-05, "val/num_eos_tokens": 279, "lr": 9.687712292719997e-06, "episode": 3136, "epoch": 0.15838383838383838, "step": 196}, {"eps": 0, "objective/kl": 15.960373878479004, "objective/entropy": -102.1280746459961, "objective/non_score_reward": -0.7980186939239502, "objective/rlhf_reward": 9.143386840820312, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0009381669806316495, "policy/clipfrac_avg": 0.015553677454590797, "loss/policy_avg": -0.6271263360977173, "loss/value_avg": 1.2400249242782593, "val/clipfrac_avg": 0.1764695793390274, "policy/entropy_avg": 0.1754164695739746, "val/ratio": 0.9885835647583008, "val/ratio_var": 5.869405867997557e-05, "val/num_eos_tokens": 110, "lr": 9.683049021365267e-06, "episode": 3152, "epoch": 0.1591919191919192, "step": 197}, {"eps": 0, "objective/kl": 11.87062931060791, "objective/entropy": -89.7190170288086, "objective/non_score_reward": -0.5935314893722534, "objective/rlhf_reward": 9.843968391418457, "objective/scores": 10.4375, "policy/approxkl_avg": 0.0008885859278962016, "policy/clipfrac_avg": 0.028107576072216034, "loss/policy_avg": -0.038626529276371, "loss/value_avg": 0.6332731246948242, "val/clipfrac_avg": 0.29144570231437683, "policy/entropy_avg": 0.11816668510437012, "val/ratio": 0.991458535194397, "val/ratio_var": 6.830303755123168e-05, "val/num_eos_tokens": 283, "lr": 9.678352329464018e-06, "episode": 3168, "epoch": 0.16, "step": 198}, {"eps": 0, "objective/kl": 11.320138931274414, "objective/entropy": -124.32833862304688, "objective/non_score_reward": -0.5660068988800049, "objective/rlhf_reward": 8.316804885864258, "objective/scores": 8.875, "policy/approxkl_avg": 0.000670759822241962, "policy/clipfrac_avg": 0.009003499522805214, "loss/policy_avg": 0.5430840849876404, "loss/value_avg": 1.1472291946411133, "val/clipfrac_avg": 0.1790837049484253, "policy/entropy_avg": 0.1954498291015625, "val/ratio": 0.9917429685592651, "val/ratio_var": 4.731408989755437e-05, "val/num_eos_tokens": 554, "lr": 9.673622250534155e-06, "episode": 3184, "epoch": 0.16080808080808082, "step": 199}, {"eps": 0, "objective/kl": 12.4759521484375, "objective/entropy": -106.52888488769531, "objective/non_score_reward": -0.623797595500946, "objective/rlhf_reward": 7.981671333312988, "objective/scores": 8.625, "policy/approxkl_avg": 0.0009805381996557117, "policy/clipfrac_avg": 0.011222105473279953, "loss/policy_avg": -0.028545048087835312, "loss/value_avg": 0.8721814155578613, "val/clipfrac_avg": 0.11438994109630585, "policy/entropy_avg": 0.1516423225402832, "val/ratio": 0.9880229830741882, "val/ratio_var": 0.0001519431680208072, "val/num_eos_tokens": 13, "lr": 9.66885881833186e-06, "episode": 3200, "epoch": 0.16161616161616163, "step": 200}, {"eps": 0, "objective/kl": 10.964399337768555, "objective/entropy": -55.07182693481445, "objective/non_score_reward": -0.5482199192047119, "objective/rlhf_reward": 8.709592819213867, "objective/scores": 9.25, "policy/approxkl_avg": 0.0027661495842039585, "policy/clipfrac_avg": 0.022680213674902916, "loss/policy_avg": 0.03965132683515549, "loss/value_avg": 0.8802922964096069, "val/clipfrac_avg": 0.2466743290424347, "policy/entropy_avg": 0.15708684921264648, "val/ratio": 0.9882746934890747, "val/ratio_var": 0.00016298489936161786, "val/num_eos_tokens": 274, "lr": 9.664062066851325e-06, "episode": 3216, "epoch": 0.16242424242424242, "step": 201}, {"eps": 0, "objective/kl": 12.811698913574219, "objective/entropy": -98.02118682861328, "objective/non_score_reward": -0.6405849456787109, "objective/rlhf_reward": 7.718790054321289, "objective/scores": 8.375, "policy/approxkl_avg": 0.0010725643951445818, "policy/clipfrac_avg": 0.010770853608846664, "loss/policy_avg": 0.31882551312446594, "loss/value_avg": 1.1626311540603638, "val/clipfrac_avg": 0.14545708894729614, "policy/entropy_avg": 0.1952628493309021, "val/ratio": 0.987186074256897, "val/ratio_var": 0.00014847182319499552, "val/num_eos_tokens": 143, "lr": 9.659232030324538e-06, "episode": 3232, "epoch": 0.16323232323232323, "step": 202}, {"eps": 0, "objective/kl": 16.778663635253906, "objective/entropy": -72.79200744628906, "objective/non_score_reward": -0.8389331698417664, "objective/rlhf_reward": 7.258723258972168, "objective/scores": 8.125, "policy/approxkl_avg": 0.0009537228033877909, "policy/clipfrac_avg": 0.0074140760116279125, "loss/policy_avg": -0.036153364926576614, "loss/value_avg": 0.21554255485534668, "val/clipfrac_avg": 0.08175989985466003, "policy/entropy_avg": 0.1759014129638672, "val/ratio": 0.9885472059249878, "val/ratio_var": 2.7167381631443277e-05, "val/num_eos_tokens": 82, "lr": 9.654368743221022e-06, "episode": 3248, "epoch": 0.16404040404040404, "step": 203}, {"eps": 0, "objective/kl": 14.448421478271484, "objective/entropy": -123.76312255859375, "objective/non_score_reward": -0.7224210500717163, "objective/rlhf_reward": 9.508047103881836, "objective/scores": 10.25, "policy/approxkl_avg": 0.0007930218707770109, "policy/clipfrac_avg": 0.010996568016707897, "loss/policy_avg": -0.25860804319381714, "loss/value_avg": 1.3965643644332886, "val/clipfrac_avg": 0.1387026160955429, "policy/entropy_avg": 0.1407322883605957, "val/ratio": 0.991324245929718, "val/ratio_var": 6.240070069907233e-05, "val/num_eos_tokens": 154, "lr": 9.649472240247588e-06, "episode": 3264, "epoch": 0.16484848484848486, "step": 204}, {"eps": 0, "objective/kl": 11.825948715209961, "objective/entropy": -65.26087951660156, "objective/non_score_reward": -0.591297447681427, "objective/rlhf_reward": 9.178234100341797, "objective/scores": 9.75, "policy/approxkl_avg": 0.0007801909814588726, "policy/clipfrac_avg": 0.005785930901765823, "loss/policy_avg": -0.58719801902771, "loss/value_avg": 1.1743042469024658, "val/clipfrac_avg": 0.25026363134384155, "policy/entropy_avg": 0.14145886898040771, "val/ratio": 0.9906296133995056, "val/ratio_var": 6.658971688011661e-05, "val/num_eos_tokens": 16, "lr": 9.6445425563481e-06, "episode": 3280, "epoch": 0.16565656565656567, "step": 205}, {"eps": 0, "objective/kl": 14.593581199645996, "objective/entropy": -76.86801147460938, "objective/non_score_reward": -0.7296790480613708, "objective/rlhf_reward": 6.567195892333984, "objective/scores": 7.3125, "policy/approxkl_avg": 0.0008751621353439987, "policy/clipfrac_avg": 0.00959798227995634, "loss/policy_avg": 0.26090189814567566, "loss/value_avg": 0.48005229234695435, "val/clipfrac_avg": 0.2793257236480713, "policy/entropy_avg": 0.16647958755493164, "val/ratio": 0.9888375997543335, "val/ratio_var": 7.126013224478811e-05, "val/num_eos_tokens": 509, "lr": 9.639579726703205e-06, "episode": 3296, "epoch": 0.16646464646464645, "step": 206}, {"eps": 0, "objective/kl": 9.699974060058594, "objective/entropy": -58.051673889160156, "objective/non_score_reward": -0.4849987328052521, "objective/rlhf_reward": 8.22984504699707, "objective/scores": 8.6875, "policy/approxkl_avg": 0.000850443379022181, "policy/clipfrac_avg": 0.008085163310170174, "loss/policy_avg": 0.10253681987524033, "loss/value_avg": 1.0820584297180176, "val/clipfrac_avg": 0.2619618773460388, "policy/entropy_avg": 0.16680526733398438, "val/ratio": 0.9909214973449707, "val/ratio_var": 6.327883602352813e-05, "val/num_eos_tokens": 61, "lr": 9.63458378673011e-06, "episode": 3312, "epoch": 0.16727272727272727, "step": 207}, {"eps": 0, "objective/kl": 14.656696319580078, "objective/entropy": -32.6884651184082, "objective/non_score_reward": -0.7328348159790039, "objective/rlhf_reward": 9.114821434020996, "objective/scores": 9.875, "policy/approxkl_avg": 0.0014994569355621934, "policy/clipfrac_avg": 0.012033997103571892, "loss/policy_avg": 0.1208038404583931, "loss/value_avg": 0.7759054899215698, "val/clipfrac_avg": 0.18890069425106049, "policy/entropy_avg": 0.22088956832885742, "val/ratio": 0.9868152737617493, "val/ratio_var": 0.000158175069373101, "val/num_eos_tokens": 364, "lr": 9.6295547720823e-06, "episode": 3328, "epoch": 0.16808080808080808, "step": 208}, {"eps": 0, "objective/kl": 15.448038101196289, "objective/entropy": -99.2800521850586, "objective/non_score_reward": -0.7724019289016724, "objective/rlhf_reward": 7.688535690307617, "objective/scores": 8.4375, "policy/approxkl_avg": 0.0008467785082757473, "policy/clipfrac_avg": 0.00758355762809515, "loss/policy_avg": 0.011440183967351913, "loss/value_avg": 0.2232610136270523, "val/clipfrac_avg": 0.05371936410665512, "policy/entropy_avg": 0.15184712409973145, "val/ratio": 0.9895342588424683, "val/ratio_var": 4.036638347315602e-05, "val/num_eos_tokens": 255, "lr": 9.624492718649306e-06, "episode": 3344, "epoch": 0.1688888888888889, "step": 209}, {"eps": 0, "objective/kl": 13.331558227539062, "objective/entropy": -74.2175521850586, "objective/non_score_reward": -0.666577935218811, "objective/rlhf_reward": 7.8334221839904785, "objective/scores": 8.5, "policy/approxkl_avg": 0.0010089495917782187, "policy/clipfrac_avg": 0.009919734671711922, "loss/policy_avg": 0.05170050263404846, "loss/value_avg": 0.7091935276985168, "val/clipfrac_avg": 0.2976298928260803, "policy/entropy_avg": 0.19567251205444336, "val/ratio": 0.9870831966400146, "val/ratio_var": 6.72663445584476e-05, "val/num_eos_tokens": 599, "lr": 9.619397662556434e-06, "episode": 3360, "epoch": 0.1696969696969697, "step": 210}, {"eps": 0, "objective/kl": 13.624598503112793, "objective/entropy": -130.67494201660156, "objective/non_score_reward": -0.6812299489974976, "objective/rlhf_reward": 9.256270408630371, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0007514972821809351, "policy/clipfrac_avg": 0.0054230415262281895, "loss/policy_avg": -0.4411523938179016, "loss/value_avg": 0.5515732765197754, "val/clipfrac_avg": 0.2941030263900757, "policy/entropy_avg": 0.15925180912017822, "val/ratio": 0.9901284575462341, "val/ratio_var": 5.974218584015034e-05, "val/num_eos_tokens": 550, "lr": 9.61426964016452e-06, "episode": 3376, "epoch": 0.1705050505050505, "step": 211}, {"eps": 0, "objective/kl": 13.889019012451172, "objective/entropy": -25.59563446044922, "objective/non_score_reward": -0.6944509744644165, "objective/rlhf_reward": 7.411017417907715, "objective/scores": 8.125, "policy/approxkl_avg": 0.0009575081057846546, "policy/clipfrac_avg": 0.009083281271159649, "loss/policy_avg": 0.004086721688508987, "loss/value_avg": 0.2312389463186264, "val/clipfrac_avg": 0.17202627658843994, "policy/entropy_avg": 0.15931987762451172, "val/ratio": 0.9893661737442017, "val/ratio_var": 3.219742211513221e-05, "val/num_eos_tokens": 129, "lr": 9.609108688069655e-06, "episode": 3392, "epoch": 0.1713131313131313, "step": 212}, {"eps": 0, "objective/kl": 16.511375427246094, "objective/entropy": -68.1807861328125, "objective/non_score_reward": -0.8255687355995178, "objective/rlhf_reward": 7.791618347167969, "objective/scores": 8.625, "policy/approxkl_avg": 0.0009854808449745178, "policy/clipfrac_avg": 0.005424494855105877, "loss/policy_avg": 0.027073439210653305, "loss/value_avg": 0.22074271738529205, "val/clipfrac_avg": 0.06424004584550858, "policy/entropy_avg": 0.17797672748565674, "val/ratio": 0.9874800443649292, "val/ratio_var": 5.017150397179648e-05, "val/num_eos_tokens": 11, "lr": 9.603914843102941e-06, "episode": 3408, "epoch": 0.17212121212121212, "step": 213}, {"eps": 0, "objective/kl": 14.358128547668457, "objective/entropy": -147.718994140625, "objective/non_score_reward": -0.7179064750671387, "objective/rlhf_reward": 9.012561798095703, "objective/scores": 9.75, "policy/approxkl_avg": 0.0008998280391097069, "policy/clipfrac_avg": 0.006018335931003094, "loss/policy_avg": -0.22705355286598206, "loss/value_avg": 0.47900986671447754, "val/clipfrac_avg": 0.08306095004081726, "policy/entropy_avg": 0.16658711433410645, "val/ratio": 0.9889283180236816, "val/ratio_var": 7.233618816826493e-05, "val/num_eos_tokens": 616, "lr": 9.598688142330216e-06, "episode": 3424, "epoch": 0.17292929292929293, "step": 214}, {"eps": 0, "objective/kl": 9.831127166748047, "objective/entropy": -91.03656768798828, "objective/non_score_reward": -0.4915563762187958, "objective/rlhf_reward": 9.992818832397461, "objective/scores": 10.5, "policy/approxkl_avg": 0.0010748449712991714, "policy/clipfrac_avg": 0.022500766441226006, "loss/policy_avg": -0.14958789944648743, "loss/value_avg": 2.3287415504455566, "val/clipfrac_avg": 0.27161675691604614, "policy/entropy_avg": 0.19421768188476562, "val/ratio": 0.9880467653274536, "val/ratio_var": 0.00019949306442867965, "val/num_eos_tokens": 249, "lr": 9.593428623051793e-06, "episode": 3440, "epoch": 0.17373737373737375, "step": 215}, {"eps": 0, "objective/kl": 11.43238639831543, "objective/entropy": -65.73722839355469, "objective/non_score_reward": -0.5716192722320557, "objective/rlhf_reward": 9.635412216186523, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0008728803368285298, "policy/clipfrac_avg": 0.016023337841033936, "loss/policy_avg": 0.2554932236671448, "loss/value_avg": 1.375806212425232, "val/clipfrac_avg": 0.12904679775238037, "policy/entropy_avg": 0.16358375549316406, "val/ratio": 0.9907691478729248, "val/ratio_var": 4.799722955795005e-05, "val/num_eos_tokens": 12, "lr": 9.588136322802194e-06, "episode": 3456, "epoch": 0.17454545454545456, "step": 216}, {"eps": 0, "objective/kl": 13.610488891601562, "objective/entropy": -72.137451171875, "objective/non_score_reward": -0.680524468421936, "objective/rlhf_reward": 8.858537673950195, "objective/scores": 9.5625, "policy/approxkl_avg": 0.0009341458207927644, "policy/clipfrac_avg": 0.006956940051168203, "loss/policy_avg": 0.39980143308639526, "loss/value_avg": 0.743119478225708, "val/clipfrac_avg": 0.10816793888807297, "policy/entropy_avg": 0.18271827697753906, "val/ratio": 0.9906838536262512, "val/ratio_var": 9.380819392390549e-05, "val/num_eos_tokens": 17, "lr": 9.582811279349881e-06, "episode": 3472, "epoch": 0.17535353535353534, "step": 217}, {"eps": 0, "objective/kl": 14.33843994140625, "objective/entropy": -89.79592895507812, "objective/non_score_reward": -0.7169219851493835, "objective/rlhf_reward": 9.439327239990234, "objective/scores": 10.125, "policy/approxkl_avg": 0.0010062719229608774, "policy/clipfrac_avg": 0.016506358981132507, "loss/policy_avg": 0.2500600814819336, "loss/value_avg": 0.9119748473167419, "val/clipfrac_avg": 0.07429616153240204, "policy/entropy_avg": 0.17433929443359375, "val/ratio": 0.9891164302825928, "val/ratio_var": 4.1760282329050824e-05, "val/num_eos_tokens": 11, "lr": 9.577453530696993e-06, "episode": 3488, "epoch": 0.17616161616161616, "step": 218}, {"eps": 0, "objective/kl": 12.968160629272461, "objective/entropy": -125.28169250488281, "objective/non_score_reward": -0.648408055305481, "objective/rlhf_reward": 7.792998313903809, "objective/scores": 8.4375, "policy/approxkl_avg": 0.0007275404059328139, "policy/clipfrac_avg": 0.00535131199285388, "loss/policy_avg": -0.24181823432445526, "loss/value_avg": 0.22863328456878662, "val/clipfrac_avg": 0.17199304699897766, "policy/entropy_avg": 0.14211547374725342, "val/ratio": 0.9909255504608154, "val/ratio_var": 6.174561713123694e-05, "val/num_eos_tokens": 123, "lr": 9.572063115079063e-06, "episode": 3504, "epoch": 0.17696969696969697, "step": 219}, {"eps": 0, "objective/kl": 9.079083442687988, "objective/entropy": -49.967041015625, "objective/non_score_reward": -0.45395419001579285, "objective/rlhf_reward": 10.452295303344727, "objective/scores": 10.875, "policy/approxkl_avg": 0.0008702286286279559, "policy/clipfrac_avg": 0.006548616103827953, "loss/policy_avg": -0.12509484589099884, "loss/value_avg": 0.6027560234069824, "val/clipfrac_avg": 0.08971220999956131, "policy/entropy_avg": 0.16074752807617188, "val/ratio": 0.9901782274246216, "val/ratio_var": 4.4974971388000995e-05, "val/num_eos_tokens": 193, "lr": 9.566640070964756e-06, "episode": 3520, "epoch": 0.17777777777777778, "step": 220}, {"eps": 0, "objective/kl": 15.149253845214844, "objective/entropy": -29.953493118286133, "objective/non_score_reward": -0.757462739944458, "objective/rlhf_reward": 8.062849998474121, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0010021317284554243, "policy/clipfrac_avg": 0.005520673934370279, "loss/policy_avg": -0.00034146755933761597, "loss/value_avg": 0.4031302332878113, "val/clipfrac_avg": 0.09643255174160004, "policy/entropy_avg": 0.19618558883666992, "val/ratio": 0.9874874353408813, "val/ratio_var": 3.06899928546045e-05, "val/num_eos_tokens": 200, "lr": 9.561184437055586e-06, "episode": 3536, "epoch": 0.1785858585858586, "step": 221}, {"eps": 0, "objective/kl": 9.775554656982422, "objective/entropy": -90.66460418701172, "objective/non_score_reward": -0.488777756690979, "objective/rlhf_reward": 11.565910339355469, "objective/scores": 12.0625, "policy/approxkl_avg": 0.000993978464975953, "policy/clipfrac_avg": 0.007861049845814705, "loss/policy_avg": -0.0959365963935852, "loss/value_avg": 2.4835777282714844, "val/clipfrac_avg": 0.22381994128227234, "policy/entropy_avg": 0.187469482421875, "val/ratio": 0.9890456795692444, "val/ratio_var": 6.582106288988143e-05, "val/num_eos_tokens": 27, "lr": 9.555696252285648e-06, "episode": 3552, "epoch": 0.17939393939393938, "step": 222}, {"eps": 0, "objective/kl": 17.497539520263672, "objective/entropy": -154.86912536621094, "objective/non_score_reward": -0.8748769760131836, "objective/rlhf_reward": 8.824341773986816, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0008605552138760686, "policy/clipfrac_avg": 0.005713596940040588, "loss/policy_avg": -0.23206055164337158, "loss/value_avg": 0.20906797051429749, "val/clipfrac_avg": 0.09188983589410782, "policy/entropy_avg": 0.17575836181640625, "val/ratio": 0.9883838891983032, "val/ratio_var": 6.594192382181063e-05, "val/num_eos_tokens": 225, "lr": 9.550175555821333e-06, "episode": 3568, "epoch": 0.1802020202020202, "step": 223}, {"eps": 0, "objective/kl": 12.197138786315918, "objective/entropy": -123.72830963134766, "objective/non_score_reward": -0.6098569631576538, "objective/rlhf_reward": 9.116704940795898, "objective/scores": 9.75, "policy/approxkl_avg": 0.0009076971909962595, "policy/clipfrac_avg": 0.022357013076543808, "loss/policy_avg": -0.0811835527420044, "loss/value_avg": 1.0116069316864014, "val/clipfrac_avg": 0.12505848705768585, "policy/entropy_avg": 0.19714951515197754, "val/ratio": 0.989097535610199, "val/ratio_var": 5.287026215228252e-05, "val/num_eos_tokens": 310, "lr": 9.544622387061055e-06, "episode": 3584, "epoch": 0.181010101010101, "step": 224}, {"eps": 0, "objective/kl": 13.420661926269531, "objective/entropy": -127.17562866210938, "objective/non_score_reward": -0.6710331439971924, "objective/rlhf_reward": 8.625842094421387, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0007886404637247324, "policy/clipfrac_avg": 0.006756953429430723, "loss/policy_avg": 0.032674357295036316, "loss/value_avg": 1.1174589395523071, "val/clipfrac_avg": 0.13264839351177216, "policy/entropy_avg": 0.1559467315673828, "val/ratio": 0.9896252155303955, "val/ratio_var": 5.247633816907182e-05, "val/num_eos_tokens": 510, "lr": 9.539036785634961e-06, "episode": 3600, "epoch": 0.18181818181818182, "step": 225}, {"eps": 0, "objective/kl": 18.100746154785156, "objective/entropy": -67.11093139648438, "objective/non_score_reward": -0.9050373435020447, "objective/rlhf_reward": 8.509025573730469, "objective/scores": 9.4375, "policy/approxkl_avg": 0.001084581483155489, "policy/clipfrac_avg": 0.008873453363776207, "loss/policy_avg": -0.2772698402404785, "loss/value_avg": 0.6462498307228088, "val/clipfrac_avg": 0.08884630352258682, "policy/entropy_avg": 0.17022252082824707, "val/ratio": 0.9877827167510986, "val/ratio_var": 4.777583308168687e-05, "val/num_eos_tokens": 123, "lr": 9.533418791404658e-06, "episode": 3616, "epoch": 0.18262626262626264, "step": 226}, {"eps": 0, "objective/kl": 15.785117149353027, "objective/entropy": -100.66576385498047, "objective/non_score_reward": -0.7892558574676514, "objective/rlhf_reward": 8.710744857788086, "objective/scores": 9.5, "policy/approxkl_avg": 0.0010630425531417131, "policy/clipfrac_avg": 0.010223347693681717, "loss/policy_avg": 0.1585109829902649, "loss/value_avg": 0.5168319940567017, "val/clipfrac_avg": 0.06462043523788452, "policy/entropy_avg": 0.199601411819458, "val/ratio": 0.9871734380722046, "val/ratio_var": 6.318103987723589e-05, "val/num_eos_tokens": 393, "lr": 9.527768444462922e-06, "episode": 3632, "epoch": 0.18343434343434342, "step": 227}, {"eps": 0, "objective/kl": 12.633371353149414, "objective/entropy": -114.29866790771484, "objective/non_score_reward": -0.6316685676574707, "objective/rlhf_reward": 8.786300659179688, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0008345604874193668, "policy/clipfrac_avg": 0.004700110759586096, "loss/policy_avg": -0.31773215532302856, "loss/value_avg": 0.8459819555282593, "val/clipfrac_avg": 0.126671701669693, "policy/entropy_avg": 0.16971874237060547, "val/ratio": 0.9897171258926392, "val/ratio_var": 5.562942897086032e-05, "val/num_eos_tokens": 116, "lr": 9.522085785133415e-06, "episode": 3648, "epoch": 0.18424242424242424, "step": 228}, {"eps": 0, "objective/kl": 12.768682479858398, "objective/entropy": -71.8100814819336, "objective/non_score_reward": -0.638434112071991, "objective/rlhf_reward": 8.955315589904785, "objective/scores": 9.625, "policy/approxkl_avg": 0.0008987415349110961, "policy/clipfrac_avg": 0.010495506227016449, "loss/policy_avg": 0.04545650631189346, "loss/value_avg": 1.421910285949707, "val/clipfrac_avg": 0.17400485277175903, "policy/entropy_avg": 0.17481613159179688, "val/ratio": 0.9892676472663879, "val/ratio_var": 7.376643043244258e-05, "val/num_eos_tokens": 326, "lr": 9.516370853970395e-06, "episode": 3664, "epoch": 0.18505050505050505, "step": 229}, {"eps": 0, "objective/kl": 13.719542503356934, "objective/entropy": -140.628662109375, "objective/non_score_reward": -0.6859771013259888, "objective/rlhf_reward": 8.6304292678833, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0007748354692012072, "policy/clipfrac_avg": 0.008184252306818962, "loss/policy_avg": 0.03276506066322327, "loss/value_avg": 0.5139911770820618, "val/clipfrac_avg": 0.15990711748600006, "policy/entropy_avg": 0.1518726348876953, "val/ratio": 0.9908199310302734, "val/ratio_var": 4.362429535831325e-05, "val/num_eos_tokens": 475, "lr": 9.510623691758428e-06, "episode": 3680, "epoch": 0.18585858585858586, "step": 230}, {"eps": 0, "objective/kl": 13.595596313476562, "objective/entropy": -112.58895874023438, "objective/non_score_reward": -0.6797798871994019, "objective/rlhf_reward": 6.999907493591309, "objective/scores": 7.6875, "policy/approxkl_avg": 0.0007951147854328156, "policy/clipfrac_avg": 0.014235245995223522, "loss/policy_avg": 0.3543701469898224, "loss/value_avg": 0.7293742895126343, "val/clipfrac_avg": 0.14041440188884735, "policy/entropy_avg": 0.13730275630950928, "val/ratio": 0.9905531406402588, "val/ratio_var": 4.2730218410724774e-05, "val/num_eos_tokens": 31, "lr": 9.504844339512096e-06, "episode": 3696, "epoch": 0.18666666666666668, "step": 231}, {"eps": 0, "objective/kl": 15.552490234375, "objective/entropy": -95.20710754394531, "objective/non_score_reward": -0.7776244878768921, "objective/rlhf_reward": 8.195032119750977, "objective/scores": 9.0, "policy/approxkl_avg": 0.0009295077761635184, "policy/clipfrac_avg": 0.0064463308081030846, "loss/policy_avg": 0.07381056249141693, "loss/value_avg": 0.374658465385437, "val/clipfrac_avg": 0.08955052495002747, "policy/entropy_avg": 0.17013072967529297, "val/ratio": 0.9885688424110413, "val/ratio_var": 4.924907989334315e-05, "val/num_eos_tokens": 338, "lr": 9.499032838475708e-06, "episode": 3712, "epoch": 0.1874747474747475, "step": 232}, {"eps": 0, "objective/kl": 11.425302505493164, "objective/entropy": -55.171142578125, "objective/non_score_reward": -0.5712651014328003, "objective/rlhf_reward": 11.71779727935791, "objective/scores": 12.3125, "policy/approxkl_avg": 0.0009924890473484993, "policy/clipfrac_avg": 0.015823353081941605, "loss/policy_avg": -0.3743169605731964, "loss/value_avg": 1.4489904642105103, "val/clipfrac_avg": 0.3463912606239319, "policy/entropy_avg": 0.15912246704101562, "val/ratio": 0.9904276132583618, "val/ratio_var": 4.2650059185689315e-05, "val/num_eos_tokens": 212, "lr": 9.493189230123e-06, "episode": 3728, "epoch": 0.18828282828282827, "step": 233}, {"eps": 0, "objective/kl": 11.218280792236328, "objective/entropy": -114.34797668457031, "objective/non_score_reward": -0.5609140396118164, "objective/rlhf_reward": 6.798460960388184, "objective/scores": 7.375, "policy/approxkl_avg": 0.0013848431408405304, "policy/clipfrac_avg": 0.013197123073041439, "loss/policy_avg": 0.1211954802274704, "loss/value_avg": 1.033839225769043, "val/clipfrac_avg": 0.2574499845504761, "policy/entropy_avg": 0.1911311149597168, "val/ratio": 0.9891800284385681, "val/ratio_var": 6.6911663452629e-05, "val/num_eos_tokens": 162, "lr": 9.48731355615684e-06, "episode": 3744, "epoch": 0.1890909090909091, "step": 234}, {"eps": 0, "objective/kl": 14.93826675415039, "objective/entropy": -180.02139282226562, "objective/non_score_reward": -0.7469133138656616, "objective/rlhf_reward": 8.26871109008789, "objective/scores": 9.0, "policy/approxkl_avg": 0.0006930871168151498, "policy/clipfrac_avg": 0.0112287737429142, "loss/policy_avg": -0.05456429719924927, "loss/value_avg": 2.0512852668762207, "val/clipfrac_avg": 0.16021409630775452, "policy/entropy_avg": 0.12185859680175781, "val/ratio": 0.9917303323745728, "val/ratio_var": 3.9941103750607e-05, "val/num_eos_tokens": 303, "lr": 9.481405858508935e-06, "episode": 3760, "epoch": 0.1898989898989899, "step": 235}, {"eps": 0, "objective/kl": 14.790599822998047, "objective/entropy": -145.46041870117188, "objective/non_score_reward": -0.7395299673080444, "objective/rlhf_reward": 10.033906936645508, "objective/scores": 10.75, "policy/approxkl_avg": 0.0008720404002815485, "policy/clipfrac_avg": 0.005699498578906059, "loss/policy_avg": -0.27526289224624634, "loss/value_avg": 0.6880919933319092, "val/clipfrac_avg": 0.061457473784685135, "policy/entropy_avg": 0.1630878448486328, "val/ratio": 0.9893093109130859, "val/ratio_var": 5.2708415751112625e-05, "val/num_eos_tokens": 592, "lr": 9.475466179339528e-06, "episode": 3776, "epoch": 0.1907070707070707, "step": 236}, {"eps": 0, "objective/kl": 14.95478630065918, "objective/entropy": -107.19807434082031, "objective/non_score_reward": -0.747739315032959, "objective/rlhf_reward": 8.982728958129883, "objective/scores": 9.75, "policy/approxkl_avg": 0.0011050915345549583, "policy/clipfrac_avg": 0.032157786190509796, "loss/policy_avg": 0.15451845526695251, "loss/value_avg": 1.8476499319076538, "val/clipfrac_avg": 0.08973570913076401, "policy/entropy_avg": 0.18476414680480957, "val/ratio": 0.9881956577301025, "val/ratio_var": 5.36524603376165e-05, "val/num_eos_tokens": 680, "lr": 9.469494561037097e-06, "episode": 3792, "epoch": 0.19151515151515153, "step": 237}, {"eps": 0, "objective/kl": 15.001608848571777, "objective/entropy": -146.70474243164062, "objective/non_score_reward": -0.7500803470611572, "objective/rlhf_reward": 7.753825664520264, "objective/scores": 8.5, "policy/approxkl_avg": 0.0008398975478485227, "policy/clipfrac_avg": 0.005974689032882452, "loss/policy_avg": -0.007887881249189377, "loss/value_avg": 0.22297966480255127, "val/clipfrac_avg": 0.08070166409015656, "policy/entropy_avg": 0.15901756286621094, "val/ratio": 0.9894462823867798, "val/ratio_var": 6.362869316944852e-05, "val/num_eos_tokens": 343, "lr": 9.463491046218058e-06, "episode": 3808, "epoch": 0.1923232323232323, "step": 238}, {"eps": 0, "objective/kl": 11.3671875, "objective/entropy": -49.81889343261719, "objective/non_score_reward": -0.568359375, "objective/rlhf_reward": 9.869140625, "objective/scores": 10.4375, "policy/approxkl_avg": 0.0011585194151848555, "policy/clipfrac_avg": 0.00720559898763895, "loss/policy_avg": 0.06090904772281647, "loss/value_avg": 0.8647578358650208, "val/clipfrac_avg": 0.1332775205373764, "policy/entropy_avg": 0.18108272552490234, "val/ratio": 0.9874423146247864, "val/ratio_var": 7.534508768003434e-05, "val/num_eos_tokens": 15, "lr": 9.457455677726447e-06, "episode": 3824, "epoch": 0.19313131313131313, "step": 239}, {"eps": 0, "objective/kl": 13.683374404907227, "objective/entropy": -92.73252868652344, "objective/non_score_reward": -0.6841686964035034, "objective/rlhf_reward": 9.847081184387207, "objective/scores": 10.5, "policy/approxkl_avg": 0.000987341278232634, "policy/clipfrac_avg": 0.011263278312981129, "loss/policy_avg": -0.0028132200241088867, "loss/value_avg": 0.47351714968681335, "val/clipfrac_avg": 0.12548519670963287, "policy/entropy_avg": 0.1565999984741211, "val/ratio": 0.9901664853096008, "val/ratio_var": 5.156149563845247e-05, "val/num_eos_tokens": 335, "lr": 9.451388498633635e-06, "episode": 3840, "epoch": 0.19393939393939394, "step": 240}, {"eps": 0, "objective/kl": 14.898351669311523, "objective/entropy": -94.36567687988281, "objective/non_score_reward": -0.744917631149292, "objective/rlhf_reward": 9.731644630432129, "objective/scores": 10.5, "policy/approxkl_avg": 0.0008329006377607584, "policy/clipfrac_avg": 0.005581840872764587, "loss/policy_avg": -0.11238707602024078, "loss/value_avg": 0.30569231510162354, "val/clipfrac_avg": 0.14762382209300995, "policy/entropy_avg": 0.16306018829345703, "val/ratio": 0.9892506003379822, "val/ratio_var": 4.0011495002545416e-05, "val/num_eos_tokens": 797, "lr": 9.445289552237998e-06, "episode": 3856, "epoch": 0.19474747474747475, "step": 241}, {"eps": 0, "objective/kl": 13.85297966003418, "objective/entropy": -108.59629821777344, "objective/non_score_reward": -0.6926490068435669, "objective/rlhf_reward": 9.354225158691406, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0008656422724016011, "policy/clipfrac_avg": 0.008920292370021343, "loss/policy_avg": 0.11825011670589447, "loss/value_avg": 0.4140797257423401, "val/clipfrac_avg": 0.2358715534210205, "policy/entropy_avg": 0.1616191864013672, "val/ratio": 0.9895873069763184, "val/ratio_var": 5.987072654534131e-05, "val/num_eos_tokens": 95, "lr": 9.439158882064623e-06, "episode": 3872, "epoch": 0.19555555555555557, "step": 242}, {"eps": 0, "objective/kl": 20.37476348876953, "objective/entropy": -158.32139587402344, "objective/non_score_reward": -1.0187382698059082, "objective/rlhf_reward": 7.76641845703125, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0008469723397865891, "policy/clipfrac_avg": 0.017248257994651794, "loss/policy_avg": -0.12114518135786057, "loss/value_avg": 0.5617895126342773, "val/clipfrac_avg": 0.18548327684402466, "policy/entropy_avg": 0.17039668560028076, "val/ratio": 0.9896106719970703, "val/ratio_var": 5.761132706538774e-05, "val/num_eos_tokens": 629, "lr": 9.432996531865001e-06, "episode": 3888, "epoch": 0.19636363636363635, "step": 243}, {"eps": 0, "objective/kl": 13.425769805908203, "objective/entropy": -76.740478515625, "objective/non_score_reward": -0.6712884902954102, "objective/rlhf_reward": 8.80527400970459, "objective/scores": 9.5, "policy/approxkl_avg": 0.0008673456613905728, "policy/clipfrac_avg": 0.010126919485628605, "loss/policy_avg": 0.11465314775705338, "loss/value_avg": 0.3883507251739502, "val/clipfrac_avg": 0.08817487955093384, "policy/entropy_avg": 0.14852988719940186, "val/ratio": 0.9893420934677124, "val/ratio_var": 4.652348070521839e-05, "val/num_eos_tokens": 105, "lr": 9.426802545616696e-06, "episode": 3904, "epoch": 0.19717171717171716, "step": 244}, {"eps": 0, "objective/kl": 14.990248680114746, "objective/entropy": -103.8277587890625, "objective/non_score_reward": -0.7495124340057373, "objective/rlhf_reward": 7.506346702575684, "objective/scores": 8.25, "policy/approxkl_avg": 0.0008764934027567506, "policy/clipfrac_avg": 0.010414833202958107, "loss/policy_avg": 0.07142084836959839, "loss/value_avg": 0.4675530195236206, "val/clipfrac_avg": 0.16199971735477448, "policy/entropy_avg": 0.17772126197814941, "val/ratio": 0.9890564680099487, "val/ratio_var": 5.0618149543879554e-05, "val/num_eos_tokens": 463, "lr": 9.420576967523049e-06, "episode": 3920, "epoch": 0.19797979797979798, "step": 245}, {"eps": 0, "objective/kl": 15.050432205200195, "objective/entropy": -68.2171401977539, "objective/non_score_reward": -0.7525216341018677, "objective/rlhf_reward": 8.286540985107422, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0013208398595452309, "policy/clipfrac_avg": 0.017676567658782005, "loss/policy_avg": 0.09155968576669693, "loss/value_avg": 1.220616102218628, "val/clipfrac_avg": 0.12387647479772568, "policy/entropy_avg": 0.2582554817199707, "val/ratio": 0.9864345192909241, "val/ratio_var": 7.634109351783991e-05, "val/num_eos_tokens": 183, "lr": 9.414319842012855e-06, "episode": 3936, "epoch": 0.1987878787878788, "step": 246}, {"eps": 0, "objective/kl": 12.254688262939453, "objective/entropy": -98.11519622802734, "objective/non_score_reward": -0.6127344369888306, "objective/rlhf_reward": 7.926328182220459, "objective/scores": 8.5625, "policy/approxkl_avg": 0.000676511088386178, "policy/clipfrac_avg": 0.004696001298725605, "loss/policy_avg": 0.10812592506408691, "loss/value_avg": 0.3838285803794861, "val/clipfrac_avg": 0.27838826179504395, "policy/entropy_avg": 0.13373851776123047, "val/ratio": 0.9912145137786865, "val/ratio_var": 3.239799843868241e-05, "val/num_eos_tokens": 74, "lr": 9.408031213740045e-06, "episode": 3952, "epoch": 0.1995959595959596, "step": 247}, {"eps": 0, "objective/kl": 14.079293251037598, "objective/entropy": -93.28247833251953, "objective/non_score_reward": -0.7039647102355957, "objective/rlhf_reward": 8.092909812927246, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0010151651222258806, "policy/clipfrac_avg": 0.010710397735238075, "loss/policy_avg": 0.2526165246963501, "loss/value_avg": 0.49756327271461487, "val/clipfrac_avg": 0.0955374613404274, "policy/entropy_avg": 0.17333316802978516, "val/ratio": 0.9887602925300598, "val/ratio_var": 6.146164378151298e-05, "val/num_eos_tokens": 329, "lr": 9.401711127583372e-06, "episode": 3968, "epoch": 0.20040404040404042, "step": 248}, {"eps": 0, "objective/kl": 15.157907485961914, "objective/entropy": -106.61834716796875, "objective/non_score_reward": -0.7578953504562378, "objective/rlhf_reward": 9.171792984008789, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0008972659707069397, "policy/clipfrac_avg": 0.017500415444374084, "loss/policy_avg": -0.3386799097061157, "loss/value_avg": 0.6449569463729858, "val/clipfrac_avg": 0.16914868354797363, "policy/entropy_avg": 0.1698697805404663, "val/ratio": 0.9900318384170532, "val/ratio_var": 6.466583727160469e-05, "val/num_eos_tokens": 107, "lr": 9.395359628646087e-06, "episode": 3984, "epoch": 0.2012121212121212, "step": 249}, {"eps": 0, "objective/kl": 15.847978591918945, "objective/entropy": -88.12916564941406, "objective/non_score_reward": -0.7923988699913025, "objective/rlhf_reward": 9.281820297241211, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0008180903387255967, "policy/clipfrac_avg": 0.0073817213997244835, "loss/policy_avg": -0.26445555686950684, "loss/value_avg": 1.824251413345337, "val/clipfrac_avg": 0.09538544714450836, "policy/entropy_avg": 0.15668296813964844, "val/ratio": 0.9894523620605469, "val/ratio_var": 4.9866332119563594e-05, "val/num_eos_tokens": 460, "lr": 9.388976762255615e-06, "episode": 4000, "epoch": 0.20202020202020202, "step": 250}, {"eps": 0, "objective/kl": 15.981526374816895, "objective/entropy": -53.91804122924805, "objective/non_score_reward": -0.7990763187408447, "objective/rlhf_reward": 7.157954692840576, "objective/scores": 7.96875, "policy/approxkl_avg": 0.0009486594935879111, "policy/clipfrac_avg": 0.008417891338467598, "loss/policy_avg": -0.03636375069618225, "loss/value_avg": 0.24154099822044373, "val/clipfrac_avg": 0.13178351521492004, "policy/entropy_avg": 0.1822967529296875, "val/ratio": 0.9880446195602417, "val/ratio_var": 4.493531378102489e-05, "val/num_eos_tokens": 305, "lr": 9.382562573963238e-06, "episode": 4016, "epoch": 0.20282828282828283, "step": 251}, {"eps": 0, "objective/kl": 8.785242080688477, "objective/entropy": -116.62461853027344, "objective/non_score_reward": -0.43926215171813965, "objective/rlhf_reward": 9.603706359863281, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0005816442426294088, "policy/clipfrac_avg": 0.004240636248141527, "loss/policy_avg": -0.04389417916536331, "loss/value_avg": 0.5461421012878418, "val/clipfrac_avg": 0.24988536536693573, "policy/entropy_avg": 0.15207290649414062, "val/ratio": 0.9932804703712463, "val/ratio_var": 4.206900121062063e-05, "val/num_eos_tokens": 497, "lr": 9.376117109543769e-06, "episode": 4032, "epoch": 0.20363636363636364, "step": 252}, {"eps": 0, "objective/kl": 11.825658798217773, "objective/entropy": -88.22281646728516, "objective/non_score_reward": -0.5912830233573914, "objective/rlhf_reward": 8.143091201782227, "objective/scores": 8.75, "policy/approxkl_avg": 0.0008585604373365641, "policy/clipfrac_avg": 0.015919335186481476, "loss/policy_avg": 0.3184441924095154, "loss/value_avg": 0.8039752244949341, "val/clipfrac_avg": 0.05827297270298004, "policy/entropy_avg": 0.1344587802886963, "val/ratio": 0.9898501634597778, "val/ratio_var": 6.264803232625127e-05, "val/num_eos_tokens": 746, "lr": 9.369640414995216e-06, "episode": 4048, "epoch": 0.20444444444444446, "step": 253}, {"eps": 0, "objective/kl": 14.882261276245117, "objective/entropy": -42.02299880981445, "objective/non_score_reward": -0.744113028049469, "objective/rlhf_reward": 8.107449531555176, "objective/scores": 8.875, "policy/approxkl_avg": 0.0009829355403780937, "policy/clipfrac_avg": 0.019861944019794464, "loss/policy_avg": 0.06932681798934937, "loss/value_avg": 0.4704505503177643, "val/clipfrac_avg": 0.09604226052761078, "policy/entropy_avg": 0.18294620513916016, "val/ratio": 0.9877536296844482, "val/ratio_var": 5.442448673420586e-05, "val/num_eos_tokens": 202, "lr": 9.363132536538466e-06, "episode": 4064, "epoch": 0.20525252525252524, "step": 254}, {"eps": 0, "objective/kl": 15.13500690460205, "objective/entropy": -132.06040954589844, "objective/non_score_reward": -0.7567504048347473, "objective/rlhf_reward": 8.192468643188477, "objective/scores": 8.9375, "policy/approxkl_avg": 0.0007790173403918743, "policy/clipfrac_avg": 0.019181344658136368, "loss/policy_avg": 0.1454492211341858, "loss/value_avg": 0.2512233257293701, "val/clipfrac_avg": 0.0804247111082077, "policy/entropy_avg": 0.13058090209960938, "val/ratio": 0.9907982349395752, "val/ratio_var": 3.778808968490921e-05, "val/num_eos_tokens": 424, "lr": 9.356593520616948e-06, "episode": 4080, "epoch": 0.20606060606060606, "step": 255}, {"eps": 0, "objective/kl": 17.12119483947754, "objective/entropy": -94.291748046875, "objective/non_score_reward": -0.8560597896575928, "objective/rlhf_reward": 8.671283721923828, "objective/scores": 9.5, "policy/approxkl_avg": 0.0009006272302940488, "policy/clipfrac_avg": 0.005755542777478695, "loss/policy_avg": -0.1870514154434204, "loss/value_avg": 0.8562079668045044, "val/clipfrac_avg": 0.09692853689193726, "policy/entropy_avg": 0.1746842861175537, "val/ratio": 0.9885848164558411, "val/ratio_var": 4.2367017158539966e-05, "val/num_eos_tokens": 798, "lr": 9.350023413896302e-06, "episode": 4096, "epoch": 0.20686868686868687, "step": 256}, {"eps": 0, "objective/kl": 19.609127044677734, "objective/entropy": -157.23556518554688, "objective/non_score_reward": -0.9804563522338867, "objective/rlhf_reward": 7.660168170928955, "objective/scores": 8.625, "policy/approxkl_avg": 0.0007378142327070236, "policy/clipfrac_avg": 0.009470398537814617, "loss/policy_avg": -0.6601881384849548, "loss/value_avg": 1.8542723655700684, "val/clipfrac_avg": 0.17326012253761292, "policy/entropy_avg": 0.17036652565002441, "val/ratio": 0.990649938583374, "val/ratio_var": 5.1252005505375564e-05, "val/num_eos_tokens": 383, "lr": 9.34342226326405e-06, "episode": 4112, "epoch": 0.20767676767676768, "step": 257}, {"eps": 0, "objective/kl": 11.775390625, "objective/entropy": -95.90520477294922, "objective/non_score_reward": -0.5887695550918579, "objective/rlhf_reward": 7.544043064117432, "objective/scores": 8.125, "policy/approxkl_avg": 0.0007710008067078888, "policy/clipfrac_avg": 0.008710110560059547, "loss/policy_avg": 0.0006968677043914795, "loss/value_avg": 0.5727497935295105, "val/clipfrac_avg": 0.20523446798324585, "policy/entropy_avg": 0.13333511352539062, "val/ratio": 0.9907342195510864, "val/ratio_var": 5.306022649165243e-05, "val/num_eos_tokens": 553, "lr": 9.336790115829255e-06, "episode": 4128, "epoch": 0.2084848484848485, "step": 258}, {"eps": 0, "objective/kl": 15.882448196411133, "objective/entropy": -180.310791015625, "objective/non_score_reward": -0.7941223382949829, "objective/rlhf_reward": 7.748846530914307, "objective/scores": 8.5625, "policy/approxkl_avg": 0.0007087202975526452, "policy/clipfrac_avg": 0.008986853063106537, "loss/policy_avg": -0.11990015208721161, "loss/value_avg": 0.18013015389442444, "val/clipfrac_avg": 0.14693328738212585, "policy/entropy_avg": 0.14745712280273438, "val/ratio": 0.9911876916885376, "val/ratio_var": 5.386102566262707e-05, "val/num_eos_tokens": 400, "lr": 9.330127018922195e-06, "episode": 4144, "epoch": 0.20929292929292928, "step": 259}, {"eps": 0, "objective/kl": 11.504565238952637, "objective/entropy": -97.93316650390625, "objective/non_score_reward": -0.5752282738685608, "objective/rlhf_reward": 10.410123825073242, "objective/scores": 11.0, "policy/approxkl_avg": 0.001048518344759941, "policy/clipfrac_avg": 0.022185876965522766, "loss/policy_avg": 0.046490609645843506, "loss/value_avg": 1.0619938373565674, "val/clipfrac_avg": 0.2877679169178009, "policy/entropy_avg": 0.16389119625091553, "val/ratio": 0.989488959312439, "val/ratio_var": 7.833181007299572e-05, "val/num_eos_tokens": 62, "lr": 9.32343302009401e-06, "episode": 4160, "epoch": 0.2101010101010101, "step": 260}, {"eps": 0, "objective/kl": 14.002493858337402, "objective/entropy": -75.23338317871094, "objective/non_score_reward": -0.7001247406005859, "objective/rlhf_reward": 10.026437759399414, "objective/scores": 10.75, "policy/approxkl_avg": 0.0009849893394857645, "policy/clipfrac_avg": 0.006675241515040398, "loss/policy_avg": -0.20035383105278015, "loss/value_avg": 0.5576207041740417, "val/clipfrac_avg": 0.16675466299057007, "policy/entropy_avg": 0.17143821716308594, "val/ratio": 0.9886298775672913, "val/ratio_var": 6.11293435213156e-05, "val/num_eos_tokens": 11, "lr": 9.316708167116377e-06, "episode": 4176, "epoch": 0.2109090909090909, "step": 261}, {"eps": 0, "objective/kl": 14.501296043395996, "objective/entropy": -105.865234375, "objective/non_score_reward": -0.7250648140907288, "objective/rlhf_reward": 10.212434768676758, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0009169665863737464, "policy/clipfrac_avg": 0.013359840959310532, "loss/policy_avg": 0.04166821762919426, "loss/value_avg": 1.0122075080871582, "val/clipfrac_avg": 0.11749286949634552, "policy/entropy_avg": 0.1592082977294922, "val/ratio": 0.9892106056213379, "val/ratio_var": 5.9744346799561754e-05, "val/num_eos_tokens": 54, "lr": 9.309952507981164e-06, "episode": 4192, "epoch": 0.21171717171717172, "step": 262}, {"eps": 0, "objective/kl": 12.704278945922852, "objective/entropy": -108.7623519897461, "objective/non_score_reward": -0.6352140307426453, "objective/rlhf_reward": 8.814004898071289, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0007622368866577744, "policy/clipfrac_avg": 0.012059643864631653, "loss/policy_avg": -0.12460523098707199, "loss/value_avg": 0.3560577630996704, "val/clipfrac_avg": 0.0886392593383789, "policy/entropy_avg": 0.1394658088684082, "val/ratio": 0.9914352893829346, "val/ratio_var": 3.335260771564208e-05, "val/num_eos_tokens": 196, "lr": 9.303166090900082e-06, "episode": 4208, "epoch": 0.21252525252525253, "step": 263}, {"eps": 0, "objective/kl": 11.76550579071045, "objective/entropy": -82.82820129394531, "objective/non_score_reward": -0.5882752537727356, "objective/rlhf_reward": 9.841412544250488, "objective/scores": 10.4375, "policy/approxkl_avg": 0.000694192829541862, "policy/clipfrac_avg": 0.007227397058159113, "loss/policy_avg": 0.3000638484954834, "loss/value_avg": 3.231374502182007, "val/clipfrac_avg": 0.07290898263454437, "policy/entropy_avg": 0.12492609024047852, "val/ratio": 0.9919120073318481, "val/ratio_var": 3.886450213030912e-05, "val/num_eos_tokens": 336, "lr": 9.296348964304351e-06, "episode": 4224, "epoch": 0.21333333333333335, "step": 264}, {"eps": 0, "objective/kl": 15.946310043334961, "objective/entropy": -161.09799194335938, "objective/non_score_reward": -0.7973154783248901, "objective/rlhf_reward": 8.17143440246582, "objective/scores": 9.0, "policy/approxkl_avg": 0.000784806557931006, "policy/clipfrac_avg": 0.005639348179101944, "loss/policy_avg": 0.1362697035074234, "loss/value_avg": 0.25763940811157227, "val/clipfrac_avg": 0.034823767840862274, "policy/entropy_avg": 0.14366579055786133, "val/ratio": 0.9907838702201843, "val/ratio_var": 4.943497697240673e-05, "val/num_eos_tokens": 285, "lr": 9.289501176844348e-06, "episode": 4240, "epoch": 0.21414141414141413, "step": 265}, {"eps": 0, "objective/kl": 18.608516693115234, "objective/entropy": -38.92912673950195, "objective/non_score_reward": -0.9304258227348328, "objective/rlhf_reward": 9.499261856079102, "objective/scores": 10.4375, "policy/approxkl_avg": 0.0010509174317121506, "policy/clipfrac_avg": 0.013766787946224213, "loss/policy_avg": -0.14799898862838745, "loss/value_avg": 1.1736255884170532, "val/clipfrac_avg": 0.16095665097236633, "policy/entropy_avg": 0.17187881469726562, "val/ratio": 0.9915172457695007, "val/ratio_var": 0.00022518265177495778, "val/num_eos_tokens": 39, "lr": 9.282622777389258e-06, "episode": 4256, "epoch": 0.21494949494949495, "step": 266}, {"eps": 0, "objective/kl": 12.634617805480957, "objective/entropy": -55.39912414550781, "objective/non_score_reward": -0.6317309141159058, "objective/rlhf_reward": 9.555768966674805, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0009083591867238283, "policy/clipfrac_avg": 0.0047781881876289845, "loss/policy_avg": -0.10845097154378891, "loss/value_avg": 1.3597133159637451, "val/clipfrac_avg": 0.17677703499794006, "policy/entropy_avg": 0.11792755126953125, "val/ratio": 0.9929987192153931, "val/ratio_var": 0.00012306030839681625, "val/num_eos_tokens": 131, "lr": 9.275713815026732e-06, "episode": 4272, "epoch": 0.21575757575757576, "step": 267}, {"eps": 0, "objective/kl": 15.504014015197754, "objective/entropy": -59.651851654052734, "objective/non_score_reward": -0.7752006649971008, "objective/rlhf_reward": 8.392767906188965, "objective/scores": 9.1875, "policy/approxkl_avg": 0.0009320800891146064, "policy/clipfrac_avg": 0.00602041557431221, "loss/policy_avg": -0.40072306990623474, "loss/value_avg": 0.6274000406265259, "val/clipfrac_avg": 0.24285002052783966, "policy/entropy_avg": 0.16248130798339844, "val/ratio": 0.9889825582504272, "val/ratio_var": 4.561135210678913e-05, "val/num_eos_tokens": 26, "lr": 9.268774339062529e-06, "episode": 4288, "epoch": 0.21656565656565657, "step": 268}, {"eps": 0, "objective/kl": 17.365135192871094, "objective/entropy": -117.23480224609375, "objective/non_score_reward": -0.8682568073272705, "objective/rlhf_reward": 8.834868431091309, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0008819050854071975, "policy/clipfrac_avg": 0.015071302652359009, "loss/policy_avg": -0.4726325273513794, "loss/value_avg": 1.0873943567276, "val/clipfrac_avg": 0.06512845307588577, "policy/entropy_avg": 0.16169071197509766, "val/ratio": 0.9895751476287842, "val/ratio_var": 6.195653259055689e-05, "val/num_eos_tokens": 367, "lr": 9.261804399020175e-06, "episode": 4304, "epoch": 0.21737373737373739, "step": 269}, {"eps": 0, "objective/kl": 14.76534652709961, "objective/entropy": -90.55671691894531, "objective/non_score_reward": -0.7382673025131226, "objective/rlhf_reward": 8.56641960144043, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0007824086351320148, "policy/clipfrac_avg": 0.005687411408871412, "loss/policy_avg": -0.2973840832710266, "loss/value_avg": 0.6133067607879639, "val/clipfrac_avg": 0.1254422813653946, "policy/entropy_avg": 0.14926570653915405, "val/ratio": 0.9903734922409058, "val/ratio_var": 4.6835259126964957e-05, "val/num_eos_tokens": 217, "lr": 9.254804044640596e-06, "episode": 4320, "epoch": 0.21818181818181817, "step": 270}, {"eps": 0, "objective/kl": 15.878630638122559, "objective/entropy": -108.28328704833984, "objective/non_score_reward": -0.7939314842224121, "objective/rlhf_reward": 7.127943515777588, "objective/scores": 7.9375, "policy/approxkl_avg": 0.0007746703922748566, "policy/clipfrac_avg": 0.020203590393066406, "loss/policy_avg": 0.10686540603637695, "loss/value_avg": 0.35223832726478577, "val/clipfrac_avg": 0.11016659438610077, "policy/entropy_avg": 0.1530628204345703, "val/ratio": 0.9914586544036865, "val/ratio_var": 4.422034908202477e-05, "val/num_eos_tokens": 448, "lr": 9.24777332588177e-06, "episode": 4336, "epoch": 0.21898989898989898, "step": 271}, {"eps": 0, "objective/kl": 16.335315704345703, "objective/entropy": -133.5970458984375, "objective/non_score_reward": -0.8167657852172852, "objective/rlhf_reward": 8.417609214782715, "objective/scores": 9.25, "policy/approxkl_avg": 0.0006824146257713437, "policy/clipfrac_avg": 0.003920457791537046, "loss/policy_avg": -0.13800494372844696, "loss/value_avg": 1.3003135919570923, "val/clipfrac_avg": 0.1513872593641281, "policy/entropy_avg": 0.12821245193481445, "val/ratio": 0.9920010566711426, "val/ratio_var": 4.987249485566281e-05, "val/num_eos_tokens": 1032, "lr": 9.240712292918375e-06, "episode": 4352, "epoch": 0.2197979797979798, "step": 272}, {"eps": 0, "objective/kl": 15.891003608703613, "objective/entropy": -69.04071807861328, "objective/non_score_reward": -0.7945501804351807, "objective/rlhf_reward": 8.232793807983398, "objective/scores": 9.0, "policy/approxkl_avg": 0.0009435138781554997, "policy/clipfrac_avg": 0.00690112542361021, "loss/policy_avg": -0.41694945096969604, "loss/value_avg": 1.5838637351989746, "val/clipfrac_avg": 0.15208563208580017, "policy/entropy_avg": 0.17545413970947266, "val/ratio": 0.9881666898727417, "val/ratio_var": 9.140658949036151e-05, "val/num_eos_tokens": 95, "lr": 9.233620996141421e-06, "episode": 4368, "epoch": 0.2206060606060606, "step": 273}, {"eps": 0, "objective/kl": 14.84071159362793, "objective/entropy": -30.563913345336914, "objective/non_score_reward": -0.7420356273651123, "objective/rlhf_reward": 9.750152587890625, "objective/scores": 10.5, "policy/approxkl_avg": 0.0008996215765364468, "policy/clipfrac_avg": 0.0041840290650725365, "loss/policy_avg": -0.23135730624198914, "loss/value_avg": 1.036683440208435, "val/clipfrac_avg": 0.09715648740530014, "policy/entropy_avg": 0.16372394561767578, "val/ratio": 0.9890379905700684, "val/ratio_var": 5.0841495976783335e-05, "val/num_eos_tokens": 201, "lr": 9.2264994861579e-06, "episode": 4384, "epoch": 0.22141414141414142, "step": 274}, {"eps": 0, "objective/kl": 19.293121337890625, "objective/entropy": -208.7753448486328, "objective/non_score_reward": -0.9646561145782471, "objective/rlhf_reward": 8.429875373840332, "objective/scores": 9.375, "policy/approxkl_avg": 0.00103966542519629, "policy/clipfrac_avg": 0.021843386813998222, "loss/policy_avg": -0.008091986179351807, "loss/value_avg": 0.7564100623130798, "val/clipfrac_avg": 0.11226043850183487, "policy/entropy_avg": 0.13889122009277344, "val/ratio": 0.9908844232559204, "val/ratio_var": 4.5354645408224314e-05, "val/num_eos_tokens": 316, "lr": 9.219347813790416e-06, "episode": 4400, "epoch": 0.2222222222222222, "step": 275}, {"eps": 0, "objective/kl": 12.331647872924805, "objective/entropy": -111.80735778808594, "objective/non_score_reward": -0.6165823936462402, "objective/rlhf_reward": 11.742793083190918, "objective/scores": 12.375, "policy/approxkl_avg": 0.0008045949507504702, "policy/clipfrac_avg": 0.008964905515313148, "loss/policy_avg": -0.012354940176010132, "loss/value_avg": 1.0118482112884521, "val/clipfrac_avg": 0.18904702365398407, "policy/entropy_avg": 0.14202404022216797, "val/ratio": 0.9908614754676819, "val/ratio_var": 6.561782356584445e-05, "val/num_eos_tokens": 618, "lr": 9.212166030076832e-06, "episode": 4416, "epoch": 0.22303030303030302, "step": 276}, {"eps": 0, "objective/kl": 16.085712432861328, "objective/entropy": -201.30746459960938, "objective/non_score_reward": -0.8042856454849243, "objective/rlhf_reward": 10.590245246887207, "objective/scores": 11.375, "policy/approxkl_avg": 0.000744504330214113, "policy/clipfrac_avg": 0.010385589674115181, "loss/policy_avg": -0.44561365246772766, "loss/value_avg": 4.015078067779541, "val/clipfrac_avg": 0.10327339172363281, "policy/entropy_avg": 0.1358182281255722, "val/ratio": 0.9907582998275757, "val/ratio_var": 6.718975055264309e-05, "val/num_eos_tokens": 695, "lr": 9.204954186269894e-06, "episode": 4432, "epoch": 0.22383838383838384, "step": 277}, {"eps": 0, "objective/kl": 19.134828567504883, "objective/entropy": -81.09490966796875, "objective/non_score_reward": -0.956741452217102, "objective/rlhf_reward": 8.371383666992188, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0009207304683513939, "policy/clipfrac_avg": 0.007468705996870995, "loss/policy_avg": -0.08938002586364746, "loss/value_avg": 0.308234304189682, "val/clipfrac_avg": 0.11933809518814087, "policy/entropy_avg": 0.16699981689453125, "val/ratio": 0.9889715909957886, "val/ratio_var": 3.133176142000593e-05, "val/num_eos_tokens": 70, "lr": 9.197712333836875e-06, "episode": 4448, "epoch": 0.22464646464646465, "step": 278}, {"eps": 0, "objective/kl": 12.764959335327148, "objective/entropy": -138.91981506347656, "objective/non_score_reward": -0.638248085975647, "objective/rlhf_reward": 9.533626556396484, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0007105986005626619, "policy/clipfrac_avg": 0.01297900639474392, "loss/policy_avg": -0.4756928086280823, "loss/value_avg": 0.47323668003082275, "val/clipfrac_avg": 0.08647473156452179, "policy/entropy_avg": 0.11971688270568848, "val/ratio": 0.992650032043457, "val/ratio_var": 3.488622314762324e-05, "val/num_eos_tokens": 596, "lr": 9.190440524459203e-06, "episode": 4464, "epoch": 0.22545454545454546, "step": 279}, {"eps": 0, "objective/kl": 16.785228729248047, "objective/entropy": -100.83489227294922, "objective/non_score_reward": -0.8392614126205444, "objective/rlhf_reward": 8.426363945007324, "objective/scores": 9.25, "policy/approxkl_avg": 0.0008614714024588466, "policy/clipfrac_avg": 0.007987331598997116, "loss/policy_avg": 0.04888363927602768, "loss/value_avg": 0.3027896583080292, "val/clipfrac_avg": 0.04620702564716339, "policy/entropy_avg": 0.13911914825439453, "val/ratio": 0.9905765056610107, "val/ratio_var": 2.7363337721908465e-05, "val/num_eos_tokens": 27, "lr": 9.1831388100321e-06, "episode": 4480, "epoch": 0.22626262626262628, "step": 280}, {"eps": 0, "objective/kl": 17.810836791992188, "objective/entropy": -116.21156311035156, "objective/non_score_reward": -0.8905417919158936, "objective/rlhf_reward": 10.671957969665527, "objective/scores": 11.5625, "policy/approxkl_avg": 0.0007619216339662671, "policy/clipfrac_avg": 0.007155691273510456, "loss/policy_avg": -0.3152393400669098, "loss/value_avg": 0.8453013896942139, "val/clipfrac_avg": 0.17372316122055054, "policy/entropy_avg": 0.1354846954345703, "val/ratio": 0.9912713766098022, "val/ratio_var": 4.6165401727193967e-05, "val/num_eos_tokens": 248, "lr": 9.175807242664194e-06, "episode": 4496, "epoch": 0.22707070707070706, "step": 281}, {"eps": 0, "objective/kl": 15.429478645324707, "objective/entropy": -123.32449340820312, "objective/non_score_reward": -0.7714740037918091, "objective/rlhf_reward": 9.443368911743164, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0007769160438328981, "policy/clipfrac_avg": 0.01072020549327135, "loss/policy_avg": 0.2918892800807953, "loss/value_avg": 1.0024628639221191, "val/clipfrac_avg": 0.1120821088552475, "policy/entropy_avg": 0.12940633296966553, "val/ratio": 0.9912656545639038, "val/ratio_var": 3.813920193351805e-05, "val/num_eos_tokens": 43, "lr": 9.168445874677168e-06, "episode": 4512, "epoch": 0.22787878787878788, "step": 282}, {"eps": 0, "objective/kl": 17.246797561645508, "objective/entropy": -79.49951171875, "objective/non_score_reward": -0.8623398542404175, "objective/rlhf_reward": 7.868128776550293, "objective/scores": 8.75, "policy/approxkl_avg": 0.0008569798083044589, "policy/clipfrac_avg": 0.028804132714867592, "loss/policy_avg": -0.23349274694919586, "loss/value_avg": 0.648268461227417, "val/clipfrac_avg": 0.1429285705089569, "policy/entropy_avg": 0.13358783721923828, "val/ratio": 0.9911713004112244, "val/ratio_var": 4.6175486204447225e-05, "val/num_eos_tokens": 444, "lr": 9.16105475860537e-06, "episode": 4528, "epoch": 0.2286868686868687, "step": 283}, {"eps": 0, "objective/kl": 22.310144424438477, "objective/entropy": -95.04164123535156, "objective/non_score_reward": -1.1155072450637817, "objective/rlhf_reward": 8.868867874145508, "objective/scores": 10.0, "policy/approxkl_avg": 0.0010480072814971209, "policy/clipfrac_avg": 0.010950624942779541, "loss/policy_avg": 0.12218321859836578, "loss/value_avg": 0.19281774759292603, "val/clipfrac_avg": 0.13048362731933594, "policy/entropy_avg": 0.16732120513916016, "val/ratio": 0.9890958070755005, "val/ratio_var": 4.823869312531315e-05, "val/num_eos_tokens": 203, "lr": 9.15363394719545e-06, "episode": 4544, "epoch": 0.2294949494949495, "step": 284}, {"eps": 0, "objective/kl": 20.045467376708984, "objective/entropy": -42.42488098144531, "objective/non_score_reward": -1.0022735595703125, "objective/rlhf_reward": 7.6539764404296875, "objective/scores": 8.625, "policy/approxkl_avg": 0.0008848682045936584, "policy/clipfrac_avg": 0.007402642630040646, "loss/policy_avg": -0.002362152561545372, "loss/value_avg": 0.5866045355796814, "val/clipfrac_avg": 0.1676260083913803, "policy/entropy_avg": 0.1446681022644043, "val/ratio": 0.9896621108055115, "val/ratio_var": 2.382449383731e-05, "val/num_eos_tokens": 66, "lr": 9.146183493405976e-06, "episode": 4560, "epoch": 0.23030303030303031, "step": 285}, {"eps": 0, "objective/kl": 16.7766056060791, "objective/entropy": -106.9161376953125, "objective/non_score_reward": -0.8388303518295288, "objective/rlhf_reward": 8.204137802124023, "objective/scores": 9.0625, "policy/approxkl_avg": 0.0008132800576277077, "policy/clipfrac_avg": 0.0094719547778368, "loss/policy_avg": -0.5626534819602966, "loss/value_avg": 1.4739155769348145, "val/clipfrac_avg": 0.1908513754606247, "policy/entropy_avg": 0.14436912536621094, "val/ratio": 0.9897047281265259, "val/ratio_var": 4.6007939090486616e-05, "val/num_eos_tokens": 222, "lr": 9.13870345040706e-06, "episode": 4576, "epoch": 0.2311111111111111, "step": 286}, {"eps": 0, "objective/kl": 13.762688636779785, "objective/entropy": -144.25914001464844, "objective/non_score_reward": -0.6881344318389893, "objective/rlhf_reward": 9.99155330657959, "objective/scores": 10.6875, "policy/approxkl_avg": 0.002054408425465226, "policy/clipfrac_avg": 0.009027201682329178, "loss/policy_avg": 0.1014481782913208, "loss/value_avg": 3.819610595703125, "val/clipfrac_avg": 0.21551817655563354, "policy/entropy_avg": 0.13915538787841797, "val/ratio": 0.9908062219619751, "val/ratio_var": 0.00011211037781322375, "val/num_eos_tokens": 491, "lr": 9.131193871579975e-06, "episode": 4592, "epoch": 0.2319191919191919, "step": 287}, {"eps": 0, "objective/kl": 18.274917602539062, "objective/entropy": -86.54766082763672, "objective/non_score_reward": -0.9137458801269531, "objective/rlhf_reward": 8.640941619873047, "objective/scores": 9.5625, "policy/approxkl_avg": 0.0010243923170492053, "policy/clipfrac_avg": 0.012742959894239902, "loss/policy_avg": -0.14434215426445007, "loss/value_avg": 1.0301947593688965, "val/clipfrac_avg": 0.09064820408821106, "policy/entropy_avg": 0.17115402221679688, "val/ratio": 0.9888273477554321, "val/ratio_var": 6.463569297920913e-05, "val/num_eos_tokens": 80, "lr": 9.12365481051678e-06, "episode": 4608, "epoch": 0.23272727272727273, "step": 288}, {"eps": 0, "objective/kl": 19.723041534423828, "objective/entropy": -80.6339111328125, "objective/non_score_reward": -0.9861521124839783, "objective/rlhf_reward": 9.892754554748535, "objective/scores": 10.875, "policy/approxkl_avg": 0.0009434971725568175, "policy/clipfrac_avg": 0.009782778099179268, "loss/policy_avg": -0.22777478396892548, "loss/value_avg": 0.4974277913570404, "val/clipfrac_avg": 0.11158281564712524, "policy/entropy_avg": 0.14695453643798828, "val/ratio": 0.989630401134491, "val/ratio_var": 4.67507925350219e-05, "val/num_eos_tokens": 422, "lr": 9.116086321019928e-06, "episode": 4624, "epoch": 0.23353535353535354, "step": 289}, {"eps": 0, "objective/kl": 13.74578857421875, "objective/entropy": -90.72955322265625, "objective/non_score_reward": -0.6872894763946533, "objective/rlhf_reward": 9.58614730834961, "objective/scores": 10.25, "policy/approxkl_avg": 0.0007829072419553995, "policy/clipfrac_avg": 0.006262246984988451, "loss/policy_avg": -0.4249142110347748, "loss/value_avg": 1.535785436630249, "val/clipfrac_avg": 0.20132911205291748, "policy/entropy_avg": 0.1394805908203125, "val/ratio": 0.9905713200569153, "val/ratio_var": 4.520507718552835e-05, "val/num_eos_tokens": 98, "lr": 9.108488457101894e-06, "episode": 4640, "epoch": 0.23434343434343435, "step": 290}, {"eps": 0, "objective/kl": 18.2149600982666, "objective/entropy": -127.0085220336914, "objective/non_score_reward": -0.9107480049133301, "objective/rlhf_reward": 9.729876518249512, "objective/scores": 10.625, "policy/approxkl_avg": 0.0009391518542543054, "policy/clipfrac_avg": 0.01690862700343132, "loss/policy_avg": -0.009618103504180908, "loss/value_avg": 1.8275914192199707, "val/clipfrac_avg": 0.1852705329656601, "policy/entropy_avg": 0.14864182472229004, "val/ratio": 0.9907850027084351, "val/ratio_var": 4.850202094530687e-05, "val/num_eos_tokens": 312, "lr": 9.10086127298478e-06, "episode": 4656, "epoch": 0.23515151515151514, "step": 291}, {"eps": 0, "objective/kl": 17.068296432495117, "objective/entropy": -121.64453887939453, "objective/non_score_reward": -0.8534148335456848, "objective/rlhf_reward": 9.349710464477539, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0010989224538207054, "policy/clipfrac_avg": 0.015641771256923676, "loss/policy_avg": -0.3173828125, "loss/value_avg": 0.705482542514801, "val/clipfrac_avg": 0.12627626955509186, "policy/entropy_avg": 0.13405227661132812, "val/ratio": 0.9925355911254883, "val/ratio_var": 4.042615182697773e-05, "val/num_eos_tokens": 655, "lr": 9.093204823099932e-06, "episode": 4672, "epoch": 0.23595959595959595, "step": 292}, {"eps": 0, "objective/kl": 15.00445556640625, "objective/entropy": -96.61104583740234, "objective/non_score_reward": -0.7502228021621704, "objective/rlhf_reward": 8.163840293884277, "objective/scores": 8.9375, "policy/approxkl_avg": 0.0008020526147447526, "policy/clipfrac_avg": 0.003557023126631975, "loss/policy_avg": 0.2536461353302002, "loss/value_avg": 0.8448231220245361, "val/clipfrac_avg": 0.2270023375749588, "policy/entropy_avg": 0.14197516441345215, "val/ratio": 0.9892786741256714, "val/ratio_var": 0.00012586927914526314, "val/num_eos_tokens": 365, "lr": 9.08551916208755e-06, "episode": 4688, "epoch": 0.23676767676767677, "step": 293}, {"eps": 0, "objective/kl": 16.631576538085938, "objective/entropy": -112.97701263427734, "objective/non_score_reward": -0.8315788507461548, "objective/rlhf_reward": 12.012170791625977, "objective/scores": 12.875, "policy/approxkl_avg": 0.0006937251891940832, "policy/clipfrac_avg": 0.004472474567592144, "loss/policy_avg": -0.4800892174243927, "loss/value_avg": 0.6021974086761475, "val/clipfrac_avg": 0.32528120279312134, "policy/entropy_avg": 0.1081966757774353, "val/ratio": 0.9927380084991455, "val/ratio_var": 4.102911407244392e-05, "val/num_eos_tokens": 138, "lr": 9.077804344796302e-06, "episode": 4704, "epoch": 0.23757575757575758, "step": 294}, {"eps": 0, "objective/kl": 15.853599548339844, "objective/entropy": -9.154275894165039, "objective/non_score_reward": -0.7926799654960632, "objective/rlhf_reward": 9.293257713317871, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0008039749227464199, "policy/clipfrac_avg": 0.006328057963401079, "loss/policy_avg": 0.05609271675348282, "loss/value_avg": 0.5157899856567383, "val/clipfrac_avg": 0.19731220602989197, "policy/entropy_avg": 0.12773895263671875, "val/ratio": 0.9908485412597656, "val/ratio_var": 4.184704812359996e-05, "val/num_eos_tokens": 17, "lr": 9.070060426282924e-06, "episode": 4720, "epoch": 0.2383838383838384, "step": 295}, {"eps": 0, "objective/kl": 13.817693710327148, "objective/entropy": -76.303466796875, "objective/non_score_reward": -0.6908847093582153, "objective/rlhf_reward": 10.176302909851074, "objective/scores": 10.875, "policy/approxkl_avg": 0.0008380039362236857, "policy/clipfrac_avg": 0.018536891788244247, "loss/policy_avg": -0.5660660862922668, "loss/value_avg": 1.659135341644287, "val/clipfrac_avg": 0.31544220447540283, "policy/entropy_avg": 0.12799948453903198, "val/ratio": 0.9921280145645142, "val/ratio_var": 4.131532114115544e-05, "val/num_eos_tokens": 343, "lr": 9.06228746181184e-06, "episode": 4736, "epoch": 0.2391919191919192, "step": 296}, {"eps": 0, "objective/kl": 22.041820526123047, "objective/entropy": -127.31668853759766, "objective/non_score_reward": -1.102090835571289, "objective/rlhf_reward": 7.948689937591553, "objective/scores": 9.0625, "policy/approxkl_avg": 0.000870460004080087, "policy/clipfrac_avg": 0.008635717444121838, "loss/policy_avg": -0.13689611852169037, "loss/value_avg": 0.2256709337234497, "val/clipfrac_avg": 0.06009959429502487, "policy/entropy_avg": 0.14861583709716797, "val/ratio": 0.9896406531333923, "val/ratio_var": 3.543778802850284e-05, "val/num_eos_tokens": 572, "lr": 9.054485506854756e-06, "episode": 4752, "epoch": 0.24, "step": 297}, {"eps": 0, "objective/kl": 17.812686920166016, "objective/entropy": -99.76239013671875, "objective/non_score_reward": -0.8906343579292297, "objective/rlhf_reward": 10.421865463256836, "objective/scores": 11.3125, "policy/approxkl_avg": 0.0007631564512848854, "policy/clipfrac_avg": 0.010343081317842007, "loss/policy_avg": -0.287416934967041, "loss/value_avg": 1.405874490737915, "val/clipfrac_avg": 0.16891875863075256, "policy/entropy_avg": 0.12606573104858398, "val/ratio": 0.9915649890899658, "val/ratio_var": 3.898761860909872e-05, "val/num_eos_tokens": 157, "lr": 9.046654617090267e-06, "episode": 4768, "epoch": 0.2408080808080808, "step": 298}, {"eps": 0, "objective/kl": 18.920724868774414, "objective/entropy": -53.73211669921875, "objective/non_score_reward": -0.9460362195968628, "objective/rlhf_reward": 8.979745864868164, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0008735101437196136, "policy/clipfrac_avg": 0.005150888580828905, "loss/policy_avg": -0.3347007930278778, "loss/value_avg": 1.4577244520187378, "val/clipfrac_avg": 0.07536730915307999, "policy/entropy_avg": 0.15177440643310547, "val/ratio": 0.9899022579193115, "val/ratio_var": 3.8399972254410386e-05, "val/num_eos_tokens": 184, "lr": 9.038794848403463e-06, "episode": 4784, "epoch": 0.24161616161616162, "step": 299}, {"eps": 0, "objective/kl": 18.138174057006836, "objective/entropy": -111.41056823730469, "objective/non_score_reward": -0.9069086313247681, "objective/rlhf_reward": 6.542309761047363, "objective/scores": 7.4375, "policy/approxkl_avg": 0.000873133132699877, "policy/clipfrac_avg": 0.021350745111703873, "loss/policy_avg": -0.16605234146118164, "loss/value_avg": 0.31406185030937195, "val/clipfrac_avg": 0.05479028448462486, "policy/entropy_avg": 0.12722575664520264, "val/ratio": 0.9915505647659302, "val/ratio_var": 4.506599725573324e-05, "val/num_eos_tokens": 420, "lr": 9.030906256885528e-06, "episode": 4800, "epoch": 0.24242424242424243, "step": 300}, {"eps": 0, "objective/kl": 20.153329849243164, "objective/entropy": -129.36647033691406, "objective/non_score_reward": -1.0076665878295898, "objective/rlhf_reward": 9.46889591217041, "objective/scores": 10.5, "policy/approxkl_avg": 0.0008897838997654617, "policy/clipfrac_avg": 0.01343005895614624, "loss/policy_avg": -0.0382872149348259, "loss/value_avg": 0.3191486597061157, "val/clipfrac_avg": 0.08865603804588318, "policy/entropy_avg": 0.13002967834472656, "val/ratio": 0.990653395652771, "val/ratio_var": 4.155634087510407e-05, "val/num_eos_tokens": 169, "lr": 9.022988898833342e-06, "episode": 4816, "epoch": 0.24323232323232324, "step": 301}, {"eps": 0, "objective/kl": 23.026803970336914, "objective/entropy": -88.79689025878906, "objective/non_score_reward": -1.1513402462005615, "objective/rlhf_reward": 8.06740951538086, "objective/scores": 9.25, "policy/approxkl_avg": 0.0007821049075573683, "policy/clipfrac_avg": 0.006361945066601038, "loss/policy_avg": -0.013936442323029041, "loss/value_avg": 0.2754359245300293, "val/clipfrac_avg": 0.08944191783666611, "policy/entropy_avg": 0.147918701171875, "val/ratio": 0.9910757541656494, "val/ratio_var": 2.4175893486244604e-05, "val/num_eos_tokens": 586, "lr": 9.015042830749075e-06, "episode": 4832, "epoch": 0.24404040404040403, "step": 302}, {"eps": 0, "objective/kl": 24.887285232543945, "objective/entropy": -153.9242401123047, "objective/non_score_reward": -1.2443642616271973, "objective/rlhf_reward": 7.620870113372803, "objective/scores": 8.875, "policy/approxkl_avg": 0.0010797552531585097, "policy/clipfrac_avg": 0.020701207220554352, "loss/policy_avg": 0.16253715753555298, "loss/value_avg": 0.3962807059288025, "val/clipfrac_avg": 0.09651350975036621, "policy/entropy_avg": 0.14539474248886108, "val/ratio": 0.9895668029785156, "val/ratio_var": 5.088368197903037e-05, "val/num_eos_tokens": 141, "lr": 9.007068109339783e-06, "episode": 4848, "epoch": 0.24484848484848484, "step": 303}, {"eps": 0, "objective/kl": 18.144006729125977, "objective/entropy": -112.62982177734375, "objective/non_score_reward": -0.9072003364562988, "objective/rlhf_reward": 9.682642936706543, "objective/scores": 10.5625, "policy/approxkl_avg": 0.0008159844437614083, "policy/clipfrac_avg": 0.01846667192876339, "loss/policy_avg": 0.3374157249927521, "loss/value_avg": 3.7259681224823, "val/clipfrac_avg": 0.17207640409469604, "policy/entropy_avg": 0.1183176040649414, "val/ratio": 0.9920910596847534, "val/ratio_var": 2.8615042538149282e-05, "val/num_eos_tokens": 604, "lr": 8.999064791517013e-06, "episode": 4864, "epoch": 0.24565656565656566, "step": 304}, {"eps": 0, "objective/kl": 21.02062225341797, "objective/entropy": -110.00303649902344, "objective/non_score_reward": -1.0510311126708984, "objective/rlhf_reward": 9.300531387329102, "objective/scores": 10.375, "policy/approxkl_avg": 0.0009549619280733168, "policy/clipfrac_avg": 0.022998899221420288, "loss/policy_avg": 0.14597804844379425, "loss/value_avg": 0.4686759412288666, "val/clipfrac_avg": 0.1400592029094696, "policy/entropy_avg": 0.14881080389022827, "val/ratio": 0.9895407557487488, "val/ratio_var": 4.939505743095651e-05, "val/num_eos_tokens": 730, "lr": 8.991032934396386e-06, "episode": 4880, "epoch": 0.24646464646464647, "step": 305}, {"eps": 0, "objective/kl": 19.597259521484375, "objective/entropy": -137.5471649169922, "objective/non_score_reward": -0.9798629283905029, "objective/rlhf_reward": 8.938105583190918, "objective/scores": 9.9375, "policy/approxkl_avg": 0.0007075261091813445, "policy/clipfrac_avg": 0.013334396295249462, "loss/policy_avg": -0.09728778898715973, "loss/value_avg": 0.6937114000320435, "val/clipfrac_avg": 0.2546675205230713, "policy/entropy_avg": 0.12405538558959961, "val/ratio": 0.9915751218795776, "val/ratio_var": 4.6088196540949866e-05, "val/num_eos_tokens": 467, "lr": 8.982972595297195e-06, "episode": 4896, "epoch": 0.24727272727272728, "step": 306}, {"eps": 0, "objective/kl": 16.863101959228516, "objective/entropy": -90.83536529541016, "objective/non_score_reward": -0.8431550860404968, "objective/rlhf_reward": 10.844345092773438, "objective/scores": 11.6875, "policy/approxkl_avg": 0.0007314659305848181, "policy/clipfrac_avg": 0.007252814248204231, "loss/policy_avg": 0.1869451254606247, "loss/value_avg": 0.5242486000061035, "val/clipfrac_avg": 0.2522747218608856, "policy/entropy_avg": 0.15680980682373047, "val/ratio": 0.9911881685256958, "val/ratio_var": 4.907942275167443e-05, "val/num_eos_tokens": 632, "lr": 8.97488383174199e-06, "episode": 4912, "epoch": 0.24808080808080807, "step": 307}, {"eps": 0, "objective/kl": 19.828670501708984, "objective/entropy": -96.56021118164062, "objective/non_score_reward": -0.9914336204528809, "objective/rlhf_reward": 8.750753402709961, "objective/scores": 9.75, "policy/approxkl_avg": 0.001207109191454947, "policy/clipfrac_avg": 0.027725519612431526, "loss/policy_avg": -0.05933833122253418, "loss/value_avg": 0.7149787545204163, "val/clipfrac_avg": 0.07701820880174637, "policy/entropy_avg": 0.14550870656967163, "val/ratio": 0.9902600049972534, "val/ratio_var": 5.600972872343846e-05, "val/num_eos_tokens": 384, "lr": 8.966766701456177e-06, "episode": 4928, "epoch": 0.24888888888888888, "step": 308}, {"eps": 0, "objective/kl": 21.63240623474121, "objective/entropy": -98.84840393066406, "objective/non_score_reward": -1.081620454788208, "objective/rlhf_reward": 11.062911033630371, "objective/scores": 12.125, "policy/approxkl_avg": 0.000988299841992557, "policy/clipfrac_avg": 0.021323896944522858, "loss/policy_avg": -0.45973384380340576, "loss/value_avg": 0.9564704895019531, "val/clipfrac_avg": 0.1754695475101471, "policy/entropy_avg": 0.15676051378250122, "val/ratio": 0.9899306893348694, "val/ratio_var": 8.19729539216496e-05, "val/num_eos_tokens": 12, "lr": 8.9586212623676e-06, "episode": 4944, "epoch": 0.2496969696969697, "step": 309}, {"eps": 0, "objective/kl": 19.034969329833984, "objective/entropy": -54.279483795166016, "objective/non_score_reward": -0.9517484903335571, "objective/rlhf_reward": 8.87637710571289, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0012411081697791815, "policy/clipfrac_avg": 0.005438921041786671, "loss/policy_avg": -0.03560265153646469, "loss/value_avg": 1.048866629600525, "val/clipfrac_avg": 0.19135892391204834, "policy/entropy_avg": 0.1564793586730957, "val/ratio": 0.9885050058364868, "val/ratio_var": 7.225138688227162e-05, "val/num_eos_tokens": 3, "lr": 8.950447572606128e-06, "episode": 4960, "epoch": 0.2505050505050505, "step": 310}, {"eps": 0, "objective/kl": 16.922504425048828, "objective/entropy": -90.46046447753906, "objective/non_score_reward": -0.8461253643035889, "objective/rlhf_reward": 8.325750350952148, "objective/scores": 9.1875, "policy/approxkl_avg": 0.0010159123921766877, "policy/clipfrac_avg": 0.01187141053378582, "loss/policy_avg": 0.022867269814014435, "loss/value_avg": 0.9407914876937866, "val/clipfrac_avg": 0.22986651957035065, "policy/entropy_avg": 0.15433311462402344, "val/ratio": 0.9906753301620483, "val/ratio_var": 6.91376335453242e-05, "val/num_eos_tokens": 578, "lr": 8.94224569050324e-06, "episode": 4976, "epoch": 0.2513131313131313, "step": 311}, {"eps": 0, "objective/kl": 18.02437400817871, "objective/entropy": -124.9230728149414, "objective/non_score_reward": -0.9012187719345093, "objective/rlhf_reward": 9.07143783569336, "objective/scores": 10.0, "policy/approxkl_avg": 0.0007548386347480118, "policy/clipfrac_avg": 0.01587068662047386, "loss/policy_avg": -0.29373523592948914, "loss/value_avg": 0.7329980731010437, "val/clipfrac_avg": 0.1042708307504654, "policy/entropy_avg": 0.12101477384567261, "val/ratio": 0.9913851022720337, "val/ratio_var": 5.3006267989985645e-05, "val/num_eos_tokens": 173, "lr": 8.93401567459161e-06, "episode": 4992, "epoch": 0.25212121212121213, "step": 312}, {"eps": 0, "objective/kl": 17.74087142944336, "objective/entropy": -52.38456344604492, "objective/non_score_reward": -0.8870435953140259, "objective/rlhf_reward": 9.171550750732422, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0007087467820383608, "policy/clipfrac_avg": 0.008573219180107117, "loss/policy_avg": 0.26103121042251587, "loss/value_avg": 0.7720507383346558, "val/clipfrac_avg": 0.165409117937088, "policy/entropy_avg": 0.12889361381530762, "val/ratio": 0.9908913373947144, "val/ratio_var": 1.9720375348697416e-05, "val/num_eos_tokens": 54, "lr": 8.92575758360469e-06, "episode": 5008, "epoch": 0.25292929292929295, "step": 313}, {"eps": 0, "objective/kl": 20.900075912475586, "objective/entropy": -135.29092407226562, "objective/non_score_reward": -1.0450037717819214, "objective/rlhf_reward": 9.150308609008789, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0006939490558579564, "policy/clipfrac_avg": 0.0060662562027573586, "loss/policy_avg": -0.0015758797526359558, "loss/value_avg": 0.2577587962150574, "val/clipfrac_avg": 0.1105986088514328, "policy/entropy_avg": 0.12897872924804688, "val/ratio": 0.9913695454597473, "val/ratio_var": 4.320408697822131e-05, "val/num_eos_tokens": 75, "lr": 8.91747147647629e-06, "episode": 5024, "epoch": 0.25373737373737376, "step": 314}, {"eps": 0, "objective/kl": 17.859722137451172, "objective/entropy": -87.80855560302734, "objective/non_score_reward": -0.8929861187934875, "objective/rlhf_reward": 10.048419952392578, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0010231503983959556, "policy/clipfrac_avg": 0.005694298073649406, "loss/policy_avg": 0.1372879296541214, "loss/value_avg": 1.6457445621490479, "val/clipfrac_avg": 0.20232242345809937, "policy/entropy_avg": 0.18284660577774048, "val/ratio": 0.9879668354988098, "val/ratio_var": 5.564819002756849e-05, "val/num_eos_tokens": 37, "lr": 8.90915741234015e-06, "episode": 5040, "epoch": 0.2545454545454545, "step": 315}, {"eps": 0, "objective/kl": 18.628372192382812, "objective/entropy": -81.0003890991211, "objective/non_score_reward": -0.9314186573028564, "objective/rlhf_reward": 8.279519081115723, "objective/scores": 9.1875, "policy/approxkl_avg": 0.0008206713246181607, "policy/clipfrac_avg": 0.005000715609639883, "loss/policy_avg": -0.03426007553935051, "loss/value_avg": 0.335923969745636, "val/clipfrac_avg": 0.08686406910419464, "policy/entropy_avg": 0.14201807975769043, "val/ratio": 0.9906059503555298, "val/ratio_var": 4.339249062468298e-05, "val/num_eos_tokens": 117, "lr": 8.900815450529536e-06, "episode": 5056, "epoch": 0.25535353535353533, "step": 316}, {"eps": 0, "objective/kl": 18.402015686035156, "objective/entropy": -72.72438049316406, "objective/non_score_reward": -0.9201008081436157, "objective/rlhf_reward": 9.064273834228516, "objective/scores": 10.0, "policy/approxkl_avg": 0.002800849499180913, "policy/clipfrac_avg": 0.018393252044916153, "loss/policy_avg": -0.08662296831607819, "loss/value_avg": 0.6432848572731018, "val/clipfrac_avg": 0.24806933104991913, "policy/entropy_avg": 0.14298725128173828, "val/ratio": 0.9901000261306763, "val/ratio_var": 6.166585080791265e-05, "val/num_eos_tokens": 137, "lr": 8.892445650576796e-06, "episode": 5072, "epoch": 0.25616161616161615, "step": 317}, {"eps": 0, "objective/kl": 18.707744598388672, "objective/entropy": -108.95730590820312, "objective/non_score_reward": -0.9353872537612915, "objective/rlhf_reward": 8.502113342285156, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0008305341470986605, "policy/clipfrac_avg": 0.028556479141116142, "loss/policy_avg": -0.31295180320739746, "loss/value_avg": 0.8796172142028809, "val/clipfrac_avg": 0.08825132250785828, "policy/entropy_avg": 0.13506406545639038, "val/ratio": 0.9902423620223999, "val/ratio_var": 4.728272688225843e-05, "val/num_eos_tokens": 314, "lr": 8.884048072212952e-06, "episode": 5088, "epoch": 0.25696969696969696, "step": 318}, {"eps": 0, "objective/kl": 22.074050903320312, "objective/entropy": -173.64369201660156, "objective/non_score_reward": -1.1037025451660156, "objective/rlhf_reward": 7.716609954833984, "objective/scores": 8.8125, "policy/approxkl_avg": 0.000958627148065716, "policy/clipfrac_avg": 0.052636995911598206, "loss/policy_avg": 0.38284558057785034, "loss/value_avg": 1.085472822189331, "val/clipfrac_avg": 0.18396008014678955, "policy/entropy_avg": 0.1379566192626953, "val/ratio": 0.9896219372749329, "val/ratio_var": 5.951917773927562e-05, "val/num_eos_tokens": 46, "lr": 8.87562277536726e-06, "episode": 5104, "epoch": 0.2577777777777778, "step": 319}, {"eps": 0, "objective/kl": 15.281889915466309, "objective/entropy": -72.43736267089844, "objective/non_score_reward": -0.7640944719314575, "objective/rlhf_reward": 11.649968147277832, "objective/scores": 12.4375, "policy/approxkl_avg": 0.001308539416640997, "policy/clipfrac_avg": 0.038986120373010635, "loss/policy_avg": -0.012532621622085571, "loss/value_avg": 2.471778154373169, "val/clipfrac_avg": 0.3568417429924011, "policy/entropy_avg": 0.1654043197631836, "val/ratio": 0.9890977144241333, "val/ratio_var": 0.0001693846279522404, "val/num_eos_tokens": 403, "lr": 8.867169820166794e-06, "episode": 5120, "epoch": 0.2585858585858586, "step": 320}, {"eps": 0, "objective/kl": 15.290900230407715, "objective/entropy": -111.93338775634766, "objective/non_score_reward": -0.7645449638366699, "objective/rlhf_reward": 8.227643013000488, "objective/scores": 9.0, "policy/approxkl_avg": 0.0012109801173210144, "policy/clipfrac_avg": 0.014719913713634014, "loss/policy_avg": 0.09475673735141754, "loss/value_avg": 1.4181828498840332, "val/clipfrac_avg": 0.23112055659294128, "policy/entropy_avg": 0.2194991111755371, "val/ratio": 0.9894407987594604, "val/ratio_var": 0.0001930692815221846, "val/num_eos_tokens": 202, "lr": 8.85868926693601e-06, "episode": 5136, "epoch": 0.2593939393939394, "step": 321}, {"eps": 0, "objective/kl": 20.265304565429688, "objective/entropy": -67.42729949951172, "objective/non_score_reward": -1.0132652521133423, "objective/rlhf_reward": 9.896890640258789, "objective/scores": 10.9375, "policy/approxkl_avg": 0.001000801450572908, "policy/clipfrac_avg": 0.013037353754043579, "loss/policy_avg": -0.03570065647363663, "loss/value_avg": 0.6953660249710083, "val/clipfrac_avg": 0.2150072306394577, "policy/entropy_avg": 0.15838980674743652, "val/ratio": 0.9900329113006592, "val/ratio_var": 4.196034205961041e-05, "val/num_eos_tokens": 316, "lr": 8.850181176196316e-06, "episode": 5152, "epoch": 0.2602020202020202, "step": 322}, {"eps": 0, "objective/kl": 20.744216918945312, "objective/entropy": -34.665523529052734, "objective/non_score_reward": -1.0372107028961182, "objective/rlhf_reward": 8.302632331848145, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0010730468202382326, "policy/clipfrac_avg": 0.00638403557240963, "loss/policy_avg": -0.10239146649837494, "loss/value_avg": 0.11446619033813477, "val/clipfrac_avg": 0.04469859600067139, "policy/entropy_avg": 0.172841876745224, "val/ratio": 0.9883930683135986, "val/ratio_var": 3.371910861460492e-05, "val/num_eos_tokens": 82, "lr": 8.84164560866564e-06, "episode": 5168, "epoch": 0.261010101010101, "step": 323}, {"eps": 0, "objective/kl": 21.241905212402344, "objective/entropy": -215.53665161132812, "objective/non_score_reward": -1.0620954036712646, "objective/rlhf_reward": 8.551185607910156, "objective/scores": 9.625, "policy/approxkl_avg": 0.000840157619677484, "policy/clipfrac_avg": 0.006537843961268663, "loss/policy_avg": -0.12644696235656738, "loss/value_avg": 0.28114426136016846, "val/clipfrac_avg": 0.04681334272027016, "policy/entropy_avg": 0.15298840403556824, "val/ratio": 0.9906848669052124, "val/ratio_var": 3.288772495579906e-05, "val/num_eos_tokens": 128, "lr": 8.833082625258003e-06, "episode": 5184, "epoch": 0.26181818181818184, "step": 324}, {"eps": 0, "objective/kl": 17.67336654663086, "objective/entropy": -130.52597045898438, "objective/non_score_reward": -0.8836683034896851, "objective/rlhf_reward": 8.811644554138184, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0012680396903306246, "policy/clipfrac_avg": 0.007880561985075474, "loss/policy_avg": 0.32401522994041443, "loss/value_avg": 0.6005299687385559, "val/clipfrac_avg": 0.19150397181510925, "policy/entropy_avg": 0.1447734832763672, "val/ratio": 0.9905928373336792, "val/ratio_var": 5.435891216620803e-05, "val/num_eos_tokens": 618, "lr": 8.824492287083072e-06, "episode": 5200, "epoch": 0.26262626262626265, "step": 325}, {"eps": 0, "objective/kl": 18.464536666870117, "objective/entropy": -127.53799438476562, "objective/non_score_reward": -0.9232268333435059, "objective/rlhf_reward": 7.912710666656494, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0008032459882088006, "policy/clipfrac_avg": 0.006318090949207544, "loss/policy_avg": -0.15733838081359863, "loss/value_avg": 0.5087392926216125, "val/clipfrac_avg": 0.08119470626115799, "policy/entropy_avg": 0.14925909042358398, "val/ratio": 0.9900246858596802, "val/ratio_var": 4.682234794017859e-05, "val/num_eos_tokens": 386, "lr": 8.815874655445742e-06, "episode": 5216, "epoch": 0.2634343434343434, "step": 326}, {"eps": 0, "objective/kl": 20.631011962890625, "objective/entropy": -80.7568359375, "objective/non_score_reward": -1.0315505266189575, "objective/rlhf_reward": 8.792668342590332, "objective/scores": 9.8125, "policy/approxkl_avg": 0.0008535610395483673, "policy/clipfrac_avg": 0.020397143438458443, "loss/policy_avg": -0.4491274952888489, "loss/value_avg": 0.6588542461395264, "val/clipfrac_avg": 0.06285601854324341, "policy/entropy_avg": 0.14399003982543945, "val/ratio": 0.9904580116271973, "val/ratio_var": 3.422212466830388e-05, "val/num_eos_tokens": 202, "lr": 8.807229791845673e-06, "episode": 5232, "epoch": 0.2642424242424242, "step": 327}, {"eps": 0, "objective/kl": 20.599206924438477, "objective/entropy": -80.94671630859375, "objective/non_score_reward": -1.0299603939056396, "objective/rlhf_reward": 9.352851867675781, "objective/scores": 10.375, "policy/approxkl_avg": 0.0009967453079298139, "policy/clipfrac_avg": 0.023027844727039337, "loss/policy_avg": -0.3479881286621094, "loss/value_avg": 1.1624462604522705, "val/clipfrac_avg": 0.16796532273292542, "policy/entropy_avg": 0.15660107135772705, "val/ratio": 0.9897423982620239, "val/ratio_var": 6.0243080952204764e-05, "val/num_eos_tokens": 565, "lr": 8.798557757976877e-06, "episode": 5248, "epoch": 0.26505050505050504, "step": 328}, {"eps": 0, "objective/kl": 20.202247619628906, "objective/entropy": -106.59388732910156, "objective/non_score_reward": -1.0101124048233032, "objective/rlhf_reward": 10.064105987548828, "objective/scores": 11.0625, "policy/approxkl_avg": 0.0008892196929082274, "policy/clipfrac_avg": 0.01402290165424347, "loss/policy_avg": -0.02400904893875122, "loss/value_avg": 1.3643267154693604, "val/clipfrac_avg": 0.11122452467679977, "policy/entropy_avg": 0.15981292724609375, "val/ratio": 0.9890484809875488, "val/ratio_var": 6.913124525453895e-05, "val/num_eos_tokens": 46, "lr": 8.789858615727266e-06, "episode": 5264, "epoch": 0.26585858585858585, "step": 329}, {"eps": 0, "objective/kl": 17.81369972229004, "objective/entropy": -62.446468353271484, "objective/non_score_reward": -0.890684962272644, "objective/rlhf_reward": 8.996033668518066, "objective/scores": 9.875, "policy/approxkl_avg": 0.0006900709122419357, "policy/clipfrac_avg": 0.004856608342379332, "loss/policy_avg": -0.06329865008592606, "loss/value_avg": 0.5471635460853577, "val/clipfrac_avg": 0.1539391279220581, "policy/entropy_avg": 0.1437549591064453, "val/ratio": 0.9915682077407837, "val/ratio_var": 2.577338455012068e-05, "val/num_eos_tokens": 7, "lr": 8.781132427178203e-06, "episode": 5280, "epoch": 0.26666666666666666, "step": 330}, {"eps": 0, "objective/kl": 19.03775405883789, "objective/entropy": -88.05587768554688, "objective/non_score_reward": -0.9518877267837524, "objective/rlhf_reward": 6.600846767425537, "objective/scores": 7.5625, "policy/approxkl_avg": 0.0008258739835582674, "policy/clipfrac_avg": 0.045845311135053635, "loss/policy_avg": 0.6682453751564026, "loss/value_avg": 2.5147206783294678, "val/clipfrac_avg": 0.24742355942726135, "policy/entropy_avg": 0.12726306915283203, "val/ratio": 0.9908438920974731, "val/ratio_var": 3.244361505494453e-05, "val/num_eos_tokens": 210, "lr": 8.772379254604074e-06, "episode": 5296, "epoch": 0.2674747474747475, "step": 331}, {"eps": 0, "objective/kl": 16.866302490234375, "objective/entropy": -166.76419067382812, "objective/non_score_reward": -0.843315064907074, "objective/rlhf_reward": 9.953559875488281, "objective/scores": 10.8125, "policy/approxkl_avg": 0.0008331661229021847, "policy/clipfrac_avg": 0.0146230673417449, "loss/policy_avg": -0.07922033965587616, "loss/value_avg": 0.5227113962173462, "val/clipfrac_avg": 0.21186737716197968, "policy/entropy_avg": 0.1378493309020996, "val/ratio": 0.9917409420013428, "val/ratio_var": 4.541428279480897e-05, "val/num_eos_tokens": 293, "lr": 8.763599160471831e-06, "episode": 5312, "epoch": 0.2682828282828283, "step": 332}, {"eps": 0, "objective/kl": 23.771095275878906, "objective/entropy": -113.41132354736328, "objective/non_score_reward": -1.1885547637939453, "objective/rlhf_reward": 8.971601486206055, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0010298178531229496, "policy/clipfrac_avg": 0.008960915729403496, "loss/policy_avg": -0.10379679501056671, "loss/value_avg": 0.22473345696926117, "val/clipfrac_avg": 0.07536853849887848, "policy/entropy_avg": 0.15825366973876953, "val/ratio": 0.9886398315429688, "val/ratio_var": 8.407732821069658e-05, "val/num_eos_tokens": 622, "lr": 8.754792207440557e-06, "episode": 5328, "epoch": 0.2690909090909091, "step": 333}, {"eps": 0, "objective/kl": 12.800091743469238, "objective/entropy": -110.30827331542969, "objective/non_score_reward": -0.640004575252533, "objective/rlhf_reward": 8.453744888305664, "objective/scores": 9.125, "policy/approxkl_avg": 0.0006719123339280486, "policy/clipfrac_avg": 0.009869431145489216, "loss/policy_avg": 0.2412254959344864, "loss/value_avg": 0.5594663619995117, "val/clipfrac_avg": 0.2574247419834137, "policy/entropy_avg": 0.10278797149658203, "val/ratio": 0.9938614368438721, "val/ratio_var": 4.6416309487540275e-05, "val/num_eos_tokens": 260, "lr": 8.745958458361011e-06, "episode": 5344, "epoch": 0.2698989898989899, "step": 334}, {"eps": 0, "objective/kl": 20.533952713012695, "objective/entropy": -132.7894287109375, "objective/non_score_reward": -1.0266976356506348, "objective/rlhf_reward": 8.508459091186523, "objective/scores": 9.5625, "policy/approxkl_avg": 0.0007548570865765214, "policy/clipfrac_avg": 0.007180739659816027, "loss/policy_avg": -0.02042115107178688, "loss/value_avg": 0.5813127756118774, "val/clipfrac_avg": 0.12942752242088318, "policy/entropy_avg": 0.14557290077209473, "val/ratio": 0.9915430545806885, "val/ratio_var": 3.247900531277992e-05, "val/num_eos_tokens": 910, "lr": 8.737097976275177e-06, "episode": 5360, "epoch": 0.27070707070707073, "step": 335}, {"eps": 0, "objective/kl": 20.93308448791504, "objective/entropy": -78.77982330322266, "objective/non_score_reward": -1.046654224395752, "objective/rlhf_reward": 9.949440002441406, "objective/scores": 11.0, "policy/approxkl_avg": 0.0010565763805061579, "policy/clipfrac_avg": 0.012101948261260986, "loss/policy_avg": -0.2686082422733307, "loss/value_avg": 1.6656913757324219, "val/clipfrac_avg": 0.12851554155349731, "policy/entropy_avg": 0.15659451484680176, "val/ratio": 0.9906046390533447, "val/ratio_var": 3.88691078114789e-05, "val/num_eos_tokens": 198, "lr": 8.728210824415829e-06, "episode": 5376, "epoch": 0.27151515151515154, "step": 336}, {"eps": 0, "objective/kl": 16.704288482666016, "objective/entropy": -131.0155487060547, "objective/non_score_reward": -0.835214376449585, "objective/rlhf_reward": 10.223379135131836, "objective/scores": 11.0625, "policy/approxkl_avg": 0.0006117980228736997, "policy/clipfrac_avg": 0.0048257140442729, "loss/policy_avg": 0.24045519530773163, "loss/value_avg": 1.6145107746124268, "val/clipfrac_avg": 0.15292316675186157, "policy/entropy_avg": 0.10847282409667969, "val/ratio": 0.9926360845565796, "val/ratio_var": 4.497100235312246e-05, "val/num_eos_tokens": 780, "lr": 8.71929706620606e-06, "episode": 5392, "epoch": 0.2723232323232323, "step": 337}, {"eps": 0, "objective/kl": 18.50728988647461, "objective/entropy": -91.27498626708984, "objective/non_score_reward": -0.9253645539283752, "objective/rlhf_reward": 10.207448959350586, "objective/scores": 11.125, "policy/approxkl_avg": 0.000849106814712286, "policy/clipfrac_avg": 0.007296066731214523, "loss/policy_avg": 0.18228551745414734, "loss/value_avg": 2.502713680267334, "val/clipfrac_avg": 0.16338565945625305, "policy/entropy_avg": 0.14929378032684326, "val/ratio": 0.9899997711181641, "val/ratio_var": 5.115730164106935e-05, "val/num_eos_tokens": 195, "lr": 8.710356765258845e-06, "episode": 5408, "epoch": 0.2731313131313131, "step": 338}, {"eps": 0, "objective/kl": 20.304767608642578, "objective/entropy": -85.34510803222656, "objective/non_score_reward": -1.0152382850646973, "objective/rlhf_reward": 10.492574691772461, "objective/scores": 11.5, "policy/approxkl_avg": 0.000908980960957706, "policy/clipfrac_avg": 0.011570381000638008, "loss/policy_avg": 0.10175848752260208, "loss/value_avg": 1.1190319061279297, "val/clipfrac_avg": 0.1429380476474762, "policy/entropy_avg": 0.16573619842529297, "val/ratio": 0.9889801144599915, "val/ratio_var": 5.0471229769755155e-05, "val/num_eos_tokens": 772, "lr": 8.701389985376578e-06, "episode": 5424, "epoch": 0.2739393939393939, "step": 339}, {"eps": 0, "objective/kl": 19.259021759033203, "objective/entropy": -81.48554229736328, "objective/non_score_reward": -0.9629510641098022, "objective/rlhf_reward": 9.72454833984375, "objective/scores": 10.6875, "policy/approxkl_avg": 0.0008599548600614071, "policy/clipfrac_avg": 0.013572474010288715, "loss/policy_avg": 0.14838971197605133, "loss/value_avg": 4.23499870300293, "val/clipfrac_avg": 0.09922881424427032, "policy/entropy_avg": 0.12827253341674805, "val/ratio": 0.9914301633834839, "val/ratio_var": 4.035350139020011e-05, "val/num_eos_tokens": 42, "lr": 8.692396790550621e-06, "episode": 5440, "epoch": 0.27474747474747474, "step": 340}, {"eps": 0, "objective/kl": 23.56014633178711, "objective/entropy": -45.80091857910156, "objective/non_score_reward": -1.1780073642730713, "objective/rlhf_reward": 9.849336624145508, "objective/scores": 11.0, "policy/approxkl_avg": 0.0010738306446000934, "policy/clipfrac_avg": 0.00759343383833766, "loss/policy_avg": -0.4641180634498596, "loss/value_avg": 0.893822193145752, "val/clipfrac_avg": 0.23792031407356262, "policy/entropy_avg": 0.19598770141601562, "val/ratio": 0.9857372641563416, "val/ratio_var": 0.00010740475408965722, "val/num_eos_tokens": 90, "lr": 8.683377244960847e-06, "episode": 5456, "epoch": 0.27555555555555555, "step": 341}, {"eps": 0, "objective/kl": 18.864389419555664, "objective/entropy": -99.86138153076172, "objective/non_score_reward": -0.9432195425033569, "objective/rlhf_reward": 9.588029861450195, "objective/scores": 10.5, "policy/approxkl_avg": 0.000636940763797611, "policy/clipfrac_avg": 0.004983609076589346, "loss/policy_avg": -0.7033554911613464, "loss/value_avg": 1.0022238492965698, "val/clipfrac_avg": 0.086562380194664, "policy/entropy_avg": 0.11473152041435242, "val/ratio": 0.9927915334701538, "val/ratio_var": 2.162680175388232e-05, "val/num_eos_tokens": 62, "lr": 8.674331412975178e-06, "episode": 5472, "epoch": 0.27636363636363637, "step": 342}, {"eps": 0, "objective/kl": 15.84041690826416, "objective/entropy": -147.96414184570312, "objective/non_score_reward": -0.7920209169387817, "objective/rlhf_reward": 11.223604202270508, "objective/scores": 12.0, "policy/approxkl_avg": 0.001866342849098146, "policy/clipfrac_avg": 0.015933256596326828, "loss/policy_avg": 0.16984277963638306, "loss/value_avg": 1.0195503234863281, "val/clipfrac_avg": 0.16014033555984497, "policy/entropy_avg": 0.12890291213989258, "val/ratio": 0.9920316934585571, "val/ratio_var": 3.811761780525558e-05, "val/num_eos_tokens": 729, "lr": 8.665259359149132e-06, "episode": 5488, "epoch": 0.2771717171717172, "step": 343}, {"eps": 0, "objective/kl": 16.89540672302246, "objective/entropy": -118.87034606933594, "objective/non_score_reward": -0.8447704315185547, "objective/rlhf_reward": 10.327104568481445, "objective/scores": 11.1875, "policy/approxkl_avg": 0.0007699039415456355, "policy/clipfrac_avg": 0.011794596910476685, "loss/policy_avg": 0.05566444247961044, "loss/value_avg": 0.40116769075393677, "val/clipfrac_avg": 0.2546975612640381, "policy/entropy_avg": 0.14203107357025146, "val/ratio": 0.9906063079833984, "val/ratio_var": 5.0702645239653066e-05, "val/num_eos_tokens": 453, "lr": 8.656161148225357e-06, "episode": 5504, "epoch": 0.277979797979798, "step": 344}, {"eps": 0, "objective/kl": 16.241445541381836, "objective/entropy": -76.3680419921875, "objective/non_score_reward": -0.8120723366737366, "objective/rlhf_reward": 11.08636474609375, "objective/scores": 11.875, "policy/approxkl_avg": 0.0007258071564137936, "policy/clipfrac_avg": 0.005320374853909016, "loss/policy_avg": -0.27140140533447266, "loss/value_avg": 1.5131020545959473, "val/clipfrac_avg": 0.14076992869377136, "policy/entropy_avg": 0.12322688102722168, "val/ratio": 0.991521954536438, "val/ratio_var": 7.228642789414153e-05, "val/num_eos_tokens": 9, "lr": 8.647036845133171e-06, "episode": 5520, "epoch": 0.2787878787878788, "step": 345}, {"eps": 0, "objective/kl": 23.168113708496094, "objective/entropy": -73.7339859008789, "objective/non_score_reward": -1.1584056615829468, "objective/rlhf_reward": 9.654094696044922, "objective/scores": 10.8125, "policy/approxkl_avg": 0.0010758393909782171, "policy/clipfrac_avg": 0.011895336210727692, "loss/policy_avg": 0.31329306960105896, "loss/value_avg": 0.5371936559677124, "val/clipfrac_avg": 0.12817031145095825, "policy/entropy_avg": 0.1218266487121582, "val/ratio": 0.9905756711959839, "val/ratio_var": 3.3087821066146716e-05, "val/num_eos_tokens": 331, "lr": 8.637886514988101e-06, "episode": 5536, "epoch": 0.2795959595959596, "step": 346}, {"eps": 0, "objective/kl": 21.344974517822266, "objective/entropy": -90.95281219482422, "objective/non_score_reward": -1.067248821258545, "objective/rlhf_reward": 11.057750701904297, "objective/scores": 12.125, "policy/approxkl_avg": 0.0006777847884222865, "policy/clipfrac_avg": 0.005223071202635765, "loss/policy_avg": 0.18582414090633392, "loss/value_avg": 0.8826534748077393, "val/clipfrac_avg": 0.27874505519866943, "policy/entropy_avg": 0.12928390502929688, "val/ratio": 0.9911484122276306, "val/ratio_var": 5.034026253269985e-05, "val/num_eos_tokens": 424, "lr": 8.62871022309141e-06, "episode": 5552, "epoch": 0.2804040404040404, "step": 347}, {"eps": 0, "objective/kl": 24.182207107543945, "objective/entropy": -148.79641723632812, "objective/non_score_reward": -1.2091102600097656, "objective/rlhf_reward": 6.923702239990234, "objective/scores": 8.125, "policy/approxkl_avg": 0.0007171371835283935, "policy/clipfrac_avg": 0.009594540111720562, "loss/policy_avg": 0.3524755835533142, "loss/value_avg": 0.5708775520324707, "val/clipfrac_avg": 0.23543015122413635, "policy/entropy_avg": 0.12054634094238281, "val/ratio": 0.9915475845336914, "val/ratio_var": 3.1630850571673363e-05, "val/num_eos_tokens": 537, "lr": 8.619508034929646e-06, "episode": 5568, "epoch": 0.2812121212121212, "step": 348}, {"eps": 0, "objective/kl": 25.371280670166016, "objective/entropy": -47.92275619506836, "objective/non_score_reward": -1.268563985824585, "objective/rlhf_reward": 9.657217979431152, "objective/scores": 10.9375, "policy/approxkl_avg": 0.0007700816495344043, "policy/clipfrac_avg": 0.004703640937805176, "loss/policy_avg": -0.014234375208616257, "loss/value_avg": 0.7844265103340149, "val/clipfrac_avg": 0.2601862847805023, "policy/entropy_avg": 0.140960693359375, "val/ratio": 0.9903233051300049, "val/ratio_var": 3.460677180555649e-05, "val/num_eos_tokens": 25, "lr": 8.610280016174157e-06, "episode": 5584, "epoch": 0.282020202020202, "step": 349}, {"eps": 0, "objective/kl": 25.339210510253906, "objective/entropy": -116.39802551269531, "objective/non_score_reward": -1.266960620880127, "objective/rlhf_reward": 8.072883605957031, "objective/scores": 9.3125, "policy/approxkl_avg": 0.000991379958577454, "policy/clipfrac_avg": 0.006371830590069294, "loss/policy_avg": -0.09183628112077713, "loss/value_avg": 0.18937213718891144, "val/clipfrac_avg": 0.04358673095703125, "policy/entropy_avg": 0.14559483528137207, "val/ratio": 0.9900184869766235, "val/ratio_var": 3.288961306679994e-05, "val/num_eos_tokens": 592, "lr": 8.601026232680634e-06, "episode": 5600, "epoch": 0.2828282828282828, "step": 350}, {"eps": 0, "objective/kl": 25.083232879638672, "objective/entropy": -138.04812622070312, "objective/non_score_reward": -1.2541615962982178, "objective/rlhf_reward": 7.921619415283203, "objective/scores": 9.1875, "policy/approxkl_avg": 0.0008497297530993819, "policy/clipfrac_avg": 0.017307590693235397, "loss/policy_avg": -0.006830994039773941, "loss/value_avg": 0.9599238038063049, "val/clipfrac_avg": 0.0904756635427475, "policy/entropy_avg": 0.14270031452178955, "val/ratio": 0.9911775588989258, "val/ratio_var": 3.714514969033189e-05, "val/num_eos_tokens": 64, "lr": 8.591746750488639e-06, "episode": 5616, "epoch": 0.28363636363636363, "step": 351}, {"eps": 0, "objective/kl": 20.58731460571289, "objective/entropy": -89.55154418945312, "objective/non_score_reward": -1.0293657779693604, "objective/rlhf_reward": 9.501884460449219, "objective/scores": 10.5, "policy/approxkl_avg": 0.004132068250328302, "policy/clipfrac_avg": 0.02699371986091137, "loss/policy_avg": -0.29971742630004883, "loss/value_avg": 1.0953255891799927, "val/clipfrac_avg": 0.18198783695697784, "policy/entropy_avg": 0.14103031158447266, "val/ratio": 0.9924070835113525, "val/ratio_var": 0.0006087828078307211, "val/num_eos_tokens": 105, "lr": 8.582441635821131e-06, "episode": 5632, "epoch": 0.28444444444444444, "step": 352}, {"eps": 0, "objective/kl": 19.544586181640625, "objective/entropy": -96.5112533569336, "objective/non_score_reward": -0.9772292971611023, "objective/rlhf_reward": 10.460270881652832, "objective/scores": 11.4375, "policy/approxkl_avg": 0.0007747088093310595, "policy/clipfrac_avg": 0.0063193682581186295, "loss/policy_avg": 0.030918851494789124, "loss/value_avg": 0.21527604758739471, "val/clipfrac_avg": 0.06936304271221161, "policy/entropy_avg": 0.12494707107543945, "val/ratio": 0.991653561592102, "val/ratio_var": 4.326908310758881e-05, "val/num_eos_tokens": 302, "lr": 8.573110955083997e-06, "episode": 5648, "epoch": 0.28525252525252526, "step": 353}, {"eps": 0, "objective/kl": 19.486572265625, "objective/entropy": -39.94751739501953, "objective/non_score_reward": -0.9743286371231079, "objective/rlhf_reward": 8.470983505249023, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0012616277672350407, "policy/clipfrac_avg": 0.011073650792241096, "loss/policy_avg": 0.08415427803993225, "loss/value_avg": 0.7643447518348694, "val/clipfrac_avg": 0.14175766706466675, "policy/entropy_avg": 0.1490478515625, "val/ratio": 0.9900429844856262, "val/ratio_var": 5.867081563337706e-05, "val/num_eos_tokens": 66, "lr": 8.563754774865574e-06, "episode": 5664, "epoch": 0.28606060606060607, "step": 354}, {"eps": 0, "objective/kl": 21.03156852722168, "objective/entropy": -91.80198669433594, "objective/non_score_reward": -1.0515785217285156, "objective/rlhf_reward": 8.647640228271484, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0007716022664681077, "policy/clipfrac_avg": 0.006859911140054464, "loss/policy_avg": -0.009337957948446274, "loss/value_avg": 0.32016801834106445, "val/clipfrac_avg": 0.10722831636667252, "policy/entropy_avg": 0.15067577362060547, "val/ratio": 0.9901145696640015, "val/ratio_var": 4.587566218106076e-05, "val/num_eos_tokens": 556, "lr": 8.554373161936176e-06, "episode": 5680, "epoch": 0.2868686868686869, "step": 355}, {"eps": 0, "objective/kl": 26.18867301940918, "objective/entropy": -147.03330993652344, "objective/non_score_reward": -1.3094336986541748, "objective/rlhf_reward": 8.801894187927246, "objective/scores": 10.125, "policy/approxkl_avg": 0.0008081612177193165, "policy/clipfrac_avg": 0.0038806938100606203, "loss/policy_avg": -0.4428892731666565, "loss/value_avg": 0.8014440536499023, "val/clipfrac_avg": 0.16974319517612457, "policy/entropy_avg": 0.15047848224639893, "val/ratio": 0.9893222451210022, "val/ratio_var": 5.328768747858703e-05, "val/num_eos_tokens": 353, "lr": 8.544966183247618e-06, "episode": 5696, "epoch": 0.2876767676767677, "step": 356}, {"eps": 0, "objective/kl": 20.14461898803711, "objective/entropy": -75.95680236816406, "objective/non_score_reward": -1.0072309970855713, "objective/rlhf_reward": 9.789644241333008, "objective/scores": 10.8125, "policy/approxkl_avg": 0.001012616790831089, "policy/clipfrac_avg": 0.00791792944073677, "loss/policy_avg": -0.06810799241065979, "loss/value_avg": 0.4220166802406311, "val/clipfrac_avg": 0.06293391436338425, "policy/entropy_avg": 0.15483474731445312, "val/ratio": 0.9901751279830933, "val/ratio_var": 6.756299990229309e-05, "val/num_eos_tokens": 70, "lr": 8.535533905932739e-06, "episode": 5712, "epoch": 0.2884848484848485, "step": 357}, {"eps": 0, "objective/kl": 17.09016227722168, "objective/entropy": -78.61557006835938, "objective/non_score_reward": -0.8545081615447998, "objective/rlhf_reward": 10.024397850036621, "objective/scores": 10.875, "policy/approxkl_avg": 0.000848946045152843, "policy/clipfrac_avg": 0.0072188605554401875, "loss/policy_avg": 0.13574761152267456, "loss/value_avg": 1.2186156511306763, "val/clipfrac_avg": 0.13289573788642883, "policy/entropy_avg": 0.14081084728240967, "val/ratio": 0.9911394715309143, "val/ratio_var": 5.175899059395306e-05, "val/num_eos_tokens": 77, "lr": 8.526076397304918e-06, "episode": 5728, "epoch": 0.28929292929292927, "step": 358}, {"eps": 0, "objective/kl": 18.7655029296875, "objective/entropy": -91.85313415527344, "objective/non_score_reward": -0.938275158405304, "objective/rlhf_reward": 7.577349662780762, "objective/scores": 8.5, "policy/approxkl_avg": 0.0012549037346616387, "policy/clipfrac_avg": 0.013728092424571514, "loss/policy_avg": 0.8527805805206299, "loss/value_avg": 1.6828011274337769, "val/clipfrac_avg": 0.40935200452804565, "policy/entropy_avg": 0.17795109748840332, "val/ratio": 0.9902350902557373, "val/ratio_var": 4.9665297410683706e-05, "val/num_eos_tokens": 160, "lr": 8.516593724857598e-06, "episode": 5744, "epoch": 0.2901010101010101, "step": 359}, {"eps": 0, "objective/kl": 19.024890899658203, "objective/entropy": -113.52433013916016, "objective/non_score_reward": -0.9512445330619812, "objective/rlhf_reward": 10.044849395751953, "objective/scores": 11.0, "policy/approxkl_avg": 0.0010559242218732834, "policy/clipfrac_avg": 0.011119586415588856, "loss/policy_avg": 0.38345038890838623, "loss/value_avg": 1.2663400173187256, "val/clipfrac_avg": 0.20008543133735657, "policy/entropy_avg": 0.15532588958740234, "val/ratio": 0.9906458854675293, "val/ratio_var": 5.071852865512483e-05, "val/num_eos_tokens": 110, "lr": 8.507085956263808e-06, "episode": 5760, "epoch": 0.2909090909090909, "step": 360}, {"eps": 0, "objective/kl": 22.22720718383789, "objective/entropy": -220.52227783203125, "objective/non_score_reward": -1.1113603115081787, "objective/rlhf_reward": 8.072233200073242, "objective/scores": 9.1875, "policy/approxkl_avg": 0.000943245948292315, "policy/clipfrac_avg": 0.01834721677005291, "loss/policy_avg": 0.0799669474363327, "loss/value_avg": 0.7555652856826782, "val/clipfrac_avg": 0.11736905574798584, "policy/entropy_avg": 0.1559591293334961, "val/ratio": 0.990176796913147, "val/ratio_var": 7.675655069760978e-05, "val/num_eos_tokens": 117, "lr": 8.49755315937567e-06, "episode": 5776, "epoch": 0.2917171717171717, "step": 361}, {"eps": 0, "objective/kl": 19.329463958740234, "objective/entropy": -127.6094970703125, "objective/non_score_reward": -0.9664731621742249, "objective/rlhf_reward": 8.408527374267578, "objective/scores": 9.375, "policy/approxkl_avg": 0.0006040905136615038, "policy/clipfrac_avg": 0.004784829448908567, "loss/policy_avg": 0.24114887416362762, "loss/value_avg": 2.6182055473327637, "val/clipfrac_avg": 0.27273598313331604, "policy/entropy_avg": 0.12860214710235596, "val/ratio": 0.9914668202400208, "val/ratio_var": 5.468882591230795e-05, "val/num_eos_tokens": 598, "lr": 8.487995402223922e-06, "episode": 5792, "epoch": 0.2925252525252525, "step": 362}, {"eps": 0, "objective/kl": 24.048336029052734, "objective/entropy": -125.06207275390625, "objective/non_score_reward": -1.2024168968200684, "objective/rlhf_reward": 8.113988876342773, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0009443694725632668, "policy/clipfrac_avg": 0.013166669756174088, "loss/policy_avg": 0.25809526443481445, "loss/value_avg": 0.4706522822380066, "val/clipfrac_avg": 0.10755910724401474, "policy/entropy_avg": 0.14909744262695312, "val/ratio": 0.990389347076416, "val/ratio_var": 2.328440132259857e-05, "val/num_eos_tokens": 628, "lr": 8.478412753017433e-06, "episode": 5808, "epoch": 0.29333333333333333, "step": 363}, {"eps": 0, "objective/kl": 28.906822204589844, "objective/entropy": -92.4540786743164, "objective/non_score_reward": -1.4453411102294922, "objective/rlhf_reward": 8.749971389770508, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0010854683350771666, "policy/clipfrac_avg": 0.027407623827457428, "loss/policy_avg": 0.15223711729049683, "loss/value_avg": 1.0644316673278809, "val/clipfrac_avg": 0.15570449829101562, "policy/entropy_avg": 0.18361949920654297, "val/ratio": 0.9883888959884644, "val/ratio_var": 3.197947080479935e-05, "val/num_eos_tokens": 6, "lr": 8.46880528014271e-06, "episode": 5824, "epoch": 0.29414141414141415, "step": 364}, {"eps": 0, "objective/kl": 27.184215545654297, "objective/entropy": -123.84210205078125, "objective/non_score_reward": -1.359210729598999, "objective/rlhf_reward": 7.44547700881958, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0008880237000994384, "policy/clipfrac_avg": 0.004617786966264248, "loss/policy_avg": 0.0015800148248672485, "loss/value_avg": 0.2736946940422058, "val/clipfrac_avg": 0.1854676902294159, "policy/entropy_avg": 0.16141414642333984, "val/ratio": 0.9891216158866882, "val/ratio_var": 4.489487400860526e-05, "val/num_eos_tokens": 360, "lr": 8.459173052163414e-06, "episode": 5840, "epoch": 0.29494949494949496, "step": 365}, {"eps": 0, "objective/kl": 24.248794555664062, "objective/entropy": -79.13996887207031, "objective/non_score_reward": -1.212439775466919, "objective/rlhf_reward": 7.16060733795166, "objective/scores": 8.375, "policy/approxkl_avg": 0.0008022888214327395, "policy/clipfrac_avg": 0.006077153142541647, "loss/policy_avg": 0.012775429524481297, "loss/value_avg": 0.48028677701950073, "val/clipfrac_avg": 0.04264098405838013, "policy/entropy_avg": 0.16005897521972656, "val/ratio": 0.9902619123458862, "val/ratio_var": 3.514832860673778e-05, "val/num_eos_tokens": 176, "lr": 8.449516137819875e-06, "episode": 5856, "epoch": 0.2957575757575758, "step": 366}, {"eps": 0, "objective/kl": 32.35683822631836, "objective/entropy": -266.6590270996094, "objective/non_score_reward": -1.6178419589996338, "objective/rlhf_reward": 8.089189529418945, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0006289628800004721, "policy/clipfrac_avg": 0.009195220656692982, "loss/policy_avg": -0.18061695992946625, "loss/value_avg": 0.539659857749939, "val/clipfrac_avg": 0.10509515553712845, "policy/entropy_avg": 0.1150360107421875, "val/ratio": 0.9929022789001465, "val/ratio_var": 3.284734339104034e-05, "val/num_eos_tokens": 414, "lr": 8.439834606028594e-06, "episode": 5872, "epoch": 0.2965656565656566, "step": 367}, {"eps": 0, "objective/kl": 26.481590270996094, "objective/entropy": -54.5378532409668, "objective/non_score_reward": -1.3240795135498047, "objective/rlhf_reward": 11.027482986450195, "objective/scores": 12.375, "policy/approxkl_avg": 0.001065750140696764, "policy/clipfrac_avg": 0.006770070642232895, "loss/policy_avg": -0.16081279516220093, "loss/value_avg": 1.9553475379943848, "val/clipfrac_avg": 0.1946491301059723, "policy/entropy_avg": 0.18307781219482422, "val/ratio": 0.9881594777107239, "val/ratio_var": 5.794196476927027e-05, "val/num_eos_tokens": 329, "lr": 8.430128525881756e-06, "episode": 5888, "epoch": 0.2973737373737374, "step": 368}, {"eps": 0, "objective/kl": 27.11490249633789, "objective/entropy": -171.61798095703125, "objective/non_score_reward": -1.3557450771331787, "objective/rlhf_reward": 7.8825364112854, "objective/scores": 9.25, "policy/approxkl_avg": 0.0009488058858551085, "policy/clipfrac_avg": 0.008429555222392082, "loss/policy_avg": -0.06483446806669235, "loss/value_avg": 0.38115546107292175, "val/clipfrac_avg": 0.09484779834747314, "policy/entropy_avg": 0.17403793334960938, "val/ratio": 0.9889323711395264, "val/ratio_var": 5.186999624129385e-05, "val/num_eos_tokens": 417, "lr": 8.420397966646732e-06, "episode": 5904, "epoch": 0.29818181818181816, "step": 369}, {"eps": 0, "objective/kl": 25.170589447021484, "objective/entropy": -61.57228088378906, "objective/non_score_reward": -1.2585294246673584, "objective/rlhf_reward": 7.518814563751221, "objective/scores": 8.75, "policy/approxkl_avg": 0.001179067767225206, "policy/clipfrac_avg": 0.0051048677414655685, "loss/policy_avg": -0.01815527118742466, "loss/value_avg": 0.25550493597984314, "val/clipfrac_avg": 0.060221657156944275, "policy/entropy_avg": 0.15404510498046875, "val/ratio": 0.9892715215682983, "val/ratio_var": 3.491001552902162e-05, "val/num_eos_tokens": 108, "lr": 8.41064299776559e-06, "episode": 5920, "epoch": 0.298989898989899, "step": 370}, {"eps": 0, "objective/kl": 25.240398406982422, "objective/entropy": -62.79341125488281, "objective/non_score_reward": -1.2620198726654053, "objective/rlhf_reward": 9.948917388916016, "objective/scores": 11.1875, "policy/approxkl_avg": 0.0009268317371606827, "policy/clipfrac_avg": 0.021911144256591797, "loss/policy_avg": -0.10499698668718338, "loss/value_avg": 0.6281431317329407, "val/clipfrac_avg": 0.10369566828012466, "policy/entropy_avg": 0.15642356872558594, "val/ratio": 0.989446759223938, "val/ratio_var": 4.36611044278834e-05, "val/num_eos_tokens": 82, "lr": 8.400863688854598e-06, "episode": 5936, "epoch": 0.2997979797979798, "step": 371}, {"eps": 0, "objective/kl": 24.834531784057617, "objective/entropy": -90.6949234008789, "objective/non_score_reward": -1.2417266368865967, "objective/rlhf_reward": 8.82468032836914, "objective/scores": 10.0625, "policy/approxkl_avg": 0.001307523576542735, "policy/clipfrac_avg": 0.011635184288024902, "loss/policy_avg": -0.3728405237197876, "loss/value_avg": 1.200590968132019, "val/clipfrac_avg": 0.09476368129253387, "policy/entropy_avg": 0.14268022775650024, "val/ratio": 0.9903372526168823, "val/ratio_var": 5.383575626183301e-05, "val/num_eos_tokens": 348, "lr": 8.391060109703725e-06, "episode": 5952, "epoch": 0.3006060606060606, "step": 372}, {"eps": 0, "objective/kl": 23.921588897705078, "objective/entropy": -77.07698822021484, "objective/non_score_reward": -1.1960794925689697, "objective/rlhf_reward": 8.25704574584961, "objective/scores": 9.4375, "policy/approxkl_avg": 0.0010654187062755227, "policy/clipfrac_avg": 0.013933488167822361, "loss/policy_avg": -0.2683873772621155, "loss/value_avg": 0.8197465538978577, "val/clipfrac_avg": 0.11054794490337372, "policy/entropy_avg": 0.16466379165649414, "val/ratio": 0.9914400577545166, "val/ratio_var": 7.72410276113078e-05, "val/num_eos_tokens": 452, "lr": 8.381232330276145e-06, "episode": 5968, "epoch": 0.3014141414141414, "step": 373}, {"eps": 0, "objective/kl": 20.35816192626953, "objective/entropy": -90.79975891113281, "objective/non_score_reward": -1.0179080963134766, "objective/rlhf_reward": 9.146154403686523, "objective/scores": 10.1875, "policy/approxkl_avg": 0.0011633289977908134, "policy/clipfrac_avg": 0.011328942142426968, "loss/policy_avg": 0.050479210913181305, "loss/value_avg": 0.2513907849788666, "val/clipfrac_avg": 0.0933825820684433, "policy/entropy_avg": 0.15205001831054688, "val/ratio": 0.98993980884552, "val/ratio_var": 3.12502306769602e-05, "val/num_eos_tokens": 292, "lr": 8.371380420707737e-06, "episode": 5984, "epoch": 0.3022222222222222, "step": 374}, {"eps": 0, "objective/kl": 19.31946563720703, "objective/entropy": -152.4559326171875, "objective/non_score_reward": -0.9659732580184937, "objective/rlhf_reward": 10.717620849609375, "objective/scores": 11.6875, "policy/approxkl_avg": 0.0006288137519732118, "policy/clipfrac_avg": 0.0182405486702919, "loss/policy_avg": -0.24270930886268616, "loss/value_avg": 1.0323677062988281, "val/clipfrac_avg": 0.2301131635904312, "policy/entropy_avg": 0.14447784423828125, "val/ratio": 0.9926867485046387, "val/ratio_var": 3.9746519178152084e-05, "val/num_eos_tokens": 572, "lr": 8.361504451306585e-06, "episode": 6000, "epoch": 0.30303030303030304, "step": 375}, {"eps": 0, "objective/kl": 19.163963317871094, "objective/entropy": -204.8240966796875, "objective/non_score_reward": -0.9581981897354126, "objective/rlhf_reward": 10.190238952636719, "objective/scores": 11.125, "policy/approxkl_avg": 0.000608496367931366, "policy/clipfrac_avg": 0.005058623850345612, "loss/policy_avg": -0.6204806566238403, "loss/value_avg": 0.6857134103775024, "val/clipfrac_avg": 0.1522846519947052, "policy/entropy_avg": 0.1286005973815918, "val/ratio": 0.9927879571914673, "val/ratio_var": 4.832358536077663e-05, "val/num_eos_tokens": 235, "lr": 8.351604492552473e-06, "episode": 6016, "epoch": 0.30383838383838385, "step": 376}, {"eps": 0, "objective/kl": 20.53563690185547, "objective/entropy": -81.702392578125, "objective/non_score_reward": -1.0267819166183472, "objective/rlhf_reward": 9.441967964172363, "objective/scores": 10.5, "policy/approxkl_avg": 0.0008746698149479926, "policy/clipfrac_avg": 0.005690821446478367, "loss/policy_avg": -0.058100730180740356, "loss/value_avg": 0.4977622628211975, "val/clipfrac_avg": 0.22529853880405426, "policy/entropy_avg": 0.1544971466064453, "val/ratio": 0.9891985654830933, "val/ratio_var": 4.927700138068758e-05, "val/num_eos_tokens": 62, "lr": 8.341680615096392e-06, "episode": 6032, "epoch": 0.30464646464646467, "step": 377}, {"eps": 0, "objective/kl": 19.327545166015625, "objective/entropy": -140.73626708984375, "objective/non_score_reward": -0.966377317905426, "objective/rlhf_reward": 10.248466491699219, "objective/scores": 11.1875, "policy/approxkl_avg": 0.0006984242936596274, "policy/clipfrac_avg": 0.004717235453426838, "loss/policy_avg": -0.5821992754936218, "loss/value_avg": 1.1654460430145264, "val/clipfrac_avg": 0.29272496700286865, "policy/entropy_avg": 0.12701988220214844, "val/ratio": 0.9922663569450378, "val/ratio_var": 5.01677677675616e-05, "val/num_eos_tokens": 51, "lr": 8.331732889760021e-06, "episode": 6048, "epoch": 0.3054545454545455, "step": 378}, {"eps": 0, "objective/kl": 20.278968811035156, "objective/entropy": -77.88591766357422, "objective/non_score_reward": -1.0139484405517578, "objective/rlhf_reward": 10.259489059448242, "objective/scores": 11.25, "policy/approxkl_avg": 0.0008548812475055456, "policy/clipfrac_avg": 0.00871540792286396, "loss/policy_avg": 0.37417128682136536, "loss/value_avg": 1.0575264692306519, "val/clipfrac_avg": 0.27003729343414307, "policy/entropy_avg": 0.14256516098976135, "val/ratio": 0.9905642867088318, "val/ratio_var": 4.364799315226264e-05, "val/num_eos_tokens": 68, "lr": 8.321761387535231e-06, "episode": 6064, "epoch": 0.30626262626262624, "step": 379}, {"eps": 0, "objective/kl": 16.528039932250977, "objective/entropy": -30.368871688842773, "objective/non_score_reward": -0.826401948928833, "objective/rlhf_reward": 8.12281608581543, "objective/scores": 8.9375, "policy/approxkl_avg": 0.0010957561898976564, "policy/clipfrac_avg": 0.008557099848985672, "loss/policy_avg": 0.1371518224477768, "loss/value_avg": 0.33893001079559326, "val/clipfrac_avg": 0.09286786615848541, "policy/entropy_avg": 0.14247894287109375, "val/ratio": 0.9898958802223206, "val/ratio_var": 2.362048689974472e-05, "val/num_eos_tokens": 117, "lr": 8.31176617958358e-06, "episode": 6080, "epoch": 0.30707070707070705, "step": 380}, {"eps": 0, "objective/kl": 23.604557037353516, "objective/entropy": -30.008560180664062, "objective/non_score_reward": -1.1802278757095337, "objective/rlhf_reward": 8.143990516662598, "objective/scores": 9.3125, "policy/approxkl_avg": 0.0010318540735170245, "policy/clipfrac_avg": 0.008254901506006718, "loss/policy_avg": 0.09494100511074066, "loss/value_avg": 0.45570412278175354, "val/clipfrac_avg": 0.060982413589954376, "policy/entropy_avg": 0.18322467803955078, "val/ratio": 0.988663375377655, "val/ratio_var": 3.0003804567968473e-05, "val/num_eos_tokens": 592, "lr": 8.301747337235798e-06, "episode": 6096, "epoch": 0.30787878787878786, "step": 381}, {"eps": 0, "objective/kl": 21.36620330810547, "objective/entropy": -116.08049011230469, "objective/non_score_reward": -1.068310022354126, "objective/rlhf_reward": 10.908252716064453, "objective/scores": 12.0, "policy/approxkl_avg": 0.0010032784193754196, "policy/clipfrac_avg": 0.007687012664973736, "loss/policy_avg": -0.1134948879480362, "loss/value_avg": 0.7391153573989868, "val/clipfrac_avg": 0.11116228997707367, "policy/entropy_avg": 0.13149738311767578, "val/ratio": 0.9909749627113342, "val/ratio_var": 3.5987435694551095e-05, "val/num_eos_tokens": 273, "lr": 8.291704931991283e-06, "episode": 6112, "epoch": 0.3086868686868687, "step": 382}, {"eps": 0, "objective/kl": 24.368152618408203, "objective/entropy": -64.46329498291016, "objective/non_score_reward": -1.2184076309204102, "objective/rlhf_reward": 7.60581111907959, "objective/scores": 8.8125, "policy/approxkl_avg": 0.0010871912818402052, "policy/clipfrac_avg": 0.009455868974328041, "loss/policy_avg": 0.26366716623306274, "loss/value_avg": 0.31152695417404175, "val/clipfrac_avg": 0.17807866632938385, "policy/entropy_avg": 0.20527267456054688, "val/ratio": 0.9889589548110962, "val/ratio_var": 4.2642153857741505e-05, "val/num_eos_tokens": 14, "lr": 8.281639035517591e-06, "episode": 6128, "epoch": 0.3094949494949495, "step": 383}, {"eps": 0, "objective/kl": 23.171056747436523, "objective/entropy": -112.77015686035156, "objective/non_score_reward": -1.158552885055542, "objective/rlhf_reward": 11.263322830200195, "objective/scores": 12.4375, "policy/approxkl_avg": 0.0009117453591898084, "policy/clipfrac_avg": 0.0171449463814497, "loss/policy_avg": -0.2564522624015808, "loss/value_avg": 1.2739346027374268, "val/clipfrac_avg": 0.2111937403678894, "policy/entropy_avg": 0.1448192596435547, "val/ratio": 0.9912400245666504, "val/ratio_var": 3.967702286900021e-05, "val/num_eos_tokens": 329, "lr": 8.271549719649923e-06, "episode": 6144, "epoch": 0.3103030303030303, "step": 384}, {"eps": 0, "objective/kl": 22.004093170166016, "objective/entropy": -142.6223602294922, "objective/non_score_reward": -1.1002047061920166, "objective/rlhf_reward": 8.985733032226562, "objective/scores": 10.0625, "policy/approxkl_avg": 0.001018032431602478, "policy/clipfrac_avg": 0.009698743000626564, "loss/policy_avg": 0.22879835963249207, "loss/value_avg": 0.6531267166137695, "val/clipfrac_avg": 0.10454542934894562, "policy/entropy_avg": 0.15434914827346802, "val/ratio": 0.9905833601951599, "val/ratio_var": 8.004864503163844e-05, "val/num_eos_tokens": 182, "lr": 8.261437056390607e-06, "episode": 6160, "epoch": 0.3111111111111111, "step": 385}, {"eps": 0, "objective/kl": 24.104801177978516, "objective/entropy": -98.07450866699219, "objective/non_score_reward": -1.20524001121521, "objective/rlhf_reward": 9.251791000366211, "objective/scores": 10.4375, "policy/approxkl_avg": 0.0008569885976612568, "policy/clipfrac_avg": 0.009619549848139286, "loss/policy_avg": 0.1902710199356079, "loss/value_avg": 0.6345528960227966, "val/clipfrac_avg": 0.04851880669593811, "policy/entropy_avg": 0.15359973907470703, "val/ratio": 0.9899508357048035, "val/ratio_var": 4.097054261364974e-05, "val/num_eos_tokens": 67, "lr": 8.251301117908595e-06, "episode": 6176, "epoch": 0.31191919191919193, "step": 386}, {"eps": 0, "objective/kl": 22.46795654296875, "objective/entropy": -110.4149398803711, "objective/non_score_reward": -1.1233978271484375, "objective/rlhf_reward": 8.353164672851562, "objective/scores": 9.5, "policy/approxkl_avg": 0.0009543322958052158, "policy/clipfrac_avg": 0.014605259522795677, "loss/policy_avg": 0.17917793989181519, "loss/value_avg": 0.36408013105392456, "val/clipfrac_avg": 0.16161151230335236, "policy/entropy_avg": 0.16858911514282227, "val/ratio": 0.9885615706443787, "val/ratio_var": 4.8216566938208416e-05, "val/num_eos_tokens": 273, "lr": 8.241141976538944e-06, "episode": 6192, "epoch": 0.31272727272727274, "step": 387}, {"eps": 0, "objective/kl": 19.66629409790039, "objective/entropy": -77.30952453613281, "objective/non_score_reward": -0.9833146333694458, "objective/rlhf_reward": 10.782310485839844, "objective/scores": 11.75, "policy/approxkl_avg": 0.0008605445618741214, "policy/clipfrac_avg": 0.010454193688929081, "loss/policy_avg": -0.27181899547576904, "loss/value_avg": 1.2867136001586914, "val/clipfrac_avg": 0.2507970631122589, "policy/entropy_avg": 0.1627795696258545, "val/ratio": 0.9896144866943359, "val/ratio_var": 5.392284947447479e-05, "val/num_eos_tokens": 135, "lr": 8.230959704782289e-06, "episode": 6208, "epoch": 0.31353535353535356, "step": 388}, {"eps": 0, "objective/kl": 24.12609100341797, "objective/entropy": -50.25094985961914, "objective/non_score_reward": -1.2063045501708984, "objective/rlhf_reward": 8.664789199829102, "objective/scores": 9.875, "policy/approxkl_avg": 0.0011184238828718662, "policy/clipfrac_avg": 0.013984918594360352, "loss/policy_avg": -0.24307014048099518, "loss/value_avg": 0.3495979309082031, "val/clipfrac_avg": 0.06929454207420349, "policy/entropy_avg": 0.1985645294189453, "val/ratio": 0.9888651371002197, "val/ratio_var": 4.258898115949705e-05, "val/num_eos_tokens": 188, "lr": 8.220754375304348e-06, "episode": 6224, "epoch": 0.31434343434343437, "step": 389}, {"eps": 0, "objective/kl": 21.656740188598633, "objective/entropy": -112.65766906738281, "objective/non_score_reward": -1.0828369855880737, "objective/rlhf_reward": 9.667162895202637, "objective/scores": 10.75, "policy/approxkl_avg": 0.000986731261946261, "policy/clipfrac_avg": 0.006748675834387541, "loss/policy_avg": -0.4267215132713318, "loss/value_avg": 0.9314700365066528, "val/clipfrac_avg": 0.14616435766220093, "policy/entropy_avg": 0.19133377075195312, "val/ratio": 0.988615870475769, "val/ratio_var": 5.443965710583143e-05, "val/num_eos_tokens": 10, "lr": 8.210526060935377e-06, "episode": 6240, "epoch": 0.3151515151515151, "step": 390}, {"eps": 0, "objective/kl": 18.75835609436035, "objective/entropy": -93.58556365966797, "objective/non_score_reward": -0.9379178285598755, "objective/rlhf_reward": 10.554269790649414, "objective/scores": 11.5, "policy/approxkl_avg": 0.0006623042281717062, "policy/clipfrac_avg": 0.010373012162744999, "loss/policy_avg": -0.27975156903266907, "loss/value_avg": 0.42042723298072815, "val/clipfrac_avg": 0.10715407133102417, "policy/entropy_avg": 0.1245126724243164, "val/ratio": 0.9925373792648315, "val/ratio_var": 3.973143975599669e-05, "val/num_eos_tokens": 243, "lr": 8.200274834669675e-06, "episode": 6256, "epoch": 0.31595959595959594, "step": 391}, {"eps": 0, "objective/kl": 22.554901123046875, "objective/entropy": -50.36949920654297, "objective/non_score_reward": -1.1277451515197754, "objective/rlhf_reward": 10.442567825317383, "objective/scores": 11.5625, "policy/approxkl_avg": 0.0012943993788212538, "policy/clipfrac_avg": 0.01801796816289425, "loss/policy_avg": -0.20366400480270386, "loss/value_avg": 0.8359952569007874, "val/clipfrac_avg": 0.15647155046463013, "policy/entropy_avg": 0.17497825622558594, "val/ratio": 0.9895493984222412, "val/ratio_var": 0.00010072851000586525, "val/num_eos_tokens": 535, "lr": 8.190000769665044e-06, "episode": 6272, "epoch": 0.31676767676767675, "step": 392}, {"eps": 0, "objective/kl": 23.501121520996094, "objective/entropy": -77.18543243408203, "objective/non_score_reward": -1.175055980682373, "objective/rlhf_reward": 9.567131042480469, "objective/scores": 10.75, "policy/approxkl_avg": 0.0009706417913548648, "policy/clipfrac_avg": 0.016374697908759117, "loss/policy_avg": -0.2513708472251892, "loss/value_avg": 0.5702548623085022, "val/clipfrac_avg": 0.16020084917545319, "policy/entropy_avg": 0.16623497009277344, "val/ratio": 0.9895526170730591, "val/ratio_var": 4.693511436926201e-05, "val/num_eos_tokens": 31, "lr": 8.179703939242276e-06, "episode": 6288, "epoch": 0.31757575757575757, "step": 393}, {"eps": 0, "objective/kl": 24.797298431396484, "objective/entropy": -76.87279510498047, "objective/non_score_reward": -1.2398649454116821, "objective/rlhf_reward": 10.158573150634766, "objective/scores": 11.375, "policy/approxkl_avg": 0.0008667815709486604, "policy/clipfrac_avg": 0.007015440613031387, "loss/policy_avg": -0.2903949022293091, "loss/value_avg": 0.7161914110183716, "val/clipfrac_avg": 0.066227987408638, "policy/entropy_avg": 0.15674304962158203, "val/ratio": 0.9892519116401672, "val/ratio_var": 6.106436921982095e-05, "val/num_eos_tokens": 344, "lr": 8.169384416884632e-06, "episode": 6304, "epoch": 0.3183838383838384, "step": 394}, {"eps": 0, "objective/kl": 21.726106643676758, "objective/entropy": -162.0740966796875, "objective/non_score_reward": -1.0863052606582642, "objective/rlhf_reward": 8.612913131713867, "objective/scores": 9.6875, "policy/approxkl_avg": 0.0006859052227810025, "policy/clipfrac_avg": 0.0042783659882843494, "loss/policy_avg": -0.030849093571305275, "loss/value_avg": 0.2917250990867615, "val/clipfrac_avg": 0.08182407170534134, "policy/entropy_avg": 0.13499069213867188, "val/ratio": 0.9912937879562378, "val/ratio_var": 2.8321417630650103e-05, "val/num_eos_tokens": 190, "lr": 8.159042276237308e-06, "episode": 6320, "epoch": 0.3191919191919192, "step": 395}, {"eps": 0, "objective/kl": 19.762617111206055, "objective/entropy": -90.22354125976562, "objective/non_score_reward": -0.9881308674812317, "objective/rlhf_reward": 10.918119430541992, "objective/scores": 11.875, "policy/approxkl_avg": 0.0008516041561961174, "policy/clipfrac_avg": 0.008233188651502132, "loss/policy_avg": -0.13514100015163422, "loss/value_avg": 2.264462947845459, "val/clipfrac_avg": 0.27357757091522217, "policy/entropy_avg": 0.1468067169189453, "val/ratio": 0.9900863766670227, "val/ratio_var": 6.002257941872813e-05, "val/num_eos_tokens": 119, "lr": 8.148677591106919e-06, "episode": 6336, "epoch": 0.32, "step": 396}, {"eps": 0, "objective/kl": 21.18748664855957, "objective/entropy": -150.10548400878906, "objective/non_score_reward": -1.0593743324279785, "objective/rlhf_reward": 10.34687614440918, "objective/scores": 11.375, "policy/approxkl_avg": 0.000792744685895741, "policy/clipfrac_avg": 0.017449360340833664, "loss/policy_avg": -0.5209656357765198, "loss/value_avg": 0.5755888819694519, "val/clipfrac_avg": 0.11896035820245743, "policy/entropy_avg": 0.1412447690963745, "val/ratio": 0.9914792776107788, "val/ratio_var": 5.1306455134181306e-05, "val/num_eos_tokens": 527, "lr": 8.138290435460969e-06, "episode": 6352, "epoch": 0.3208080808080808, "step": 397}, {"eps": 0, "objective/kl": 22.376163482666016, "objective/entropy": -149.72897338867188, "objective/non_score_reward": -1.1188081502914429, "objective/rlhf_reward": 10.240567207336426, "objective/scores": 11.375, "policy/approxkl_avg": 0.0007364015327766538, "policy/clipfrac_avg": 0.008705372922122478, "loss/policy_avg": -0.40576979517936707, "loss/value_avg": 0.33822232484817505, "val/clipfrac_avg": 0.09983381628990173, "policy/entropy_avg": 0.13206863403320312, "val/ratio": 0.9914937019348145, "val/ratio_var": 3.097344233538024e-05, "val/num_eos_tokens": 92, "lr": 8.12788088342732e-06, "episode": 6368, "epoch": 0.32161616161616163, "step": 398}, {"eps": 0, "objective/kl": 18.716007232666016, "objective/entropy": -94.70143127441406, "objective/non_score_reward": -0.9358003735542297, "objective/rlhf_reward": 9.927480697631836, "objective/scores": 10.875, "policy/approxkl_avg": 0.0008329761330969632, "policy/clipfrac_avg": 0.010793356224894524, "loss/policy_avg": -0.29814594984054565, "loss/value_avg": 0.8969987034797668, "val/clipfrac_avg": 0.11709762364625931, "policy/entropy_avg": 0.1497507095336914, "val/ratio": 0.990269660949707, "val/ratio_var": 4.7837544116191566e-05, "val/num_eos_tokens": 115, "lr": 8.117449009293668e-06, "episode": 6384, "epoch": 0.32242424242424245, "step": 399}, {"eps": 0, "objective/kl": 21.73818588256836, "objective/entropy": -178.26353454589844, "objective/non_score_reward": -1.0869094133377075, "objective/rlhf_reward": 10.709965705871582, "objective/scores": 11.8125, "policy/approxkl_avg": 0.0007762957829982042, "policy/clipfrac_avg": 0.004946828819811344, "loss/policy_avg": -1.01192045211792, "loss/value_avg": 1.0538803339004517, "val/clipfrac_avg": 0.1204109638929367, "policy/entropy_avg": 0.1459665298461914, "val/ratio": 0.9914135932922363, "val/ratio_var": 3.936345819965936e-05, "val/num_eos_tokens": 105, "lr": 8.10699488750701e-06, "episode": 6400, "epoch": 0.32323232323232326, "step": 400}, {"eps": 0, "objective/kl": 22.773834228515625, "objective/entropy": -49.77229309082031, "objective/non_score_reward": -1.138691782951355, "objective/rlhf_reward": 8.751934051513672, "objective/scores": 9.875, "policy/approxkl_avg": 0.0009881152072921395, "policy/clipfrac_avg": 0.006775183603167534, "loss/policy_avg": 0.007637389004230499, "loss/value_avg": 0.6951870918273926, "val/clipfrac_avg": 0.1905401647090912, "policy/entropy_avg": 0.18160057067871094, "val/ratio": 0.9880608320236206, "val/ratio_var": 3.765714427572675e-05, "val/num_eos_tokens": 9, "lr": 8.096518592673113e-06, "episode": 6416, "epoch": 0.324040404040404, "step": 401}, {"eps": 0, "objective/kl": 22.062274932861328, "objective/entropy": -94.79197692871094, "objective/non_score_reward": -1.1031136512756348, "objective/rlhf_reward": 11.678136825561523, "objective/scores": 12.75, "policy/approxkl_avg": 0.0007612872868776321, "policy/clipfrac_avg": 0.008526662364602089, "loss/policy_avg": -0.21244871616363525, "loss/value_avg": 0.5243785381317139, "val/clipfrac_avg": 0.25014567375183105, "policy/entropy_avg": 0.16756629943847656, "val/ratio": 0.9909036159515381, "val/ratio_var": 3.402508809813298e-05, "val/num_eos_tokens": 163, "lr": 8.08602019955598e-06, "episode": 6432, "epoch": 0.32484848484848483, "step": 402}, {"eps": 0, "objective/kl": 18.592615127563477, "objective/entropy": -119.76268005371094, "objective/non_score_reward": -0.9296307563781738, "objective/rlhf_reward": 9.375057220458984, "objective/scores": 10.3125, "policy/approxkl_avg": 0.0009499984444119036, "policy/clipfrac_avg": 0.00850006751716137, "loss/policy_avg": 0.06411070376634598, "loss/value_avg": 0.7002878189086914, "val/clipfrac_avg": 0.06237310171127319, "policy/entropy_avg": 0.1573188304901123, "val/ratio": 0.9920597076416016, "val/ratio_var": 3.0845607398077846e-05, "val/num_eos_tokens": 601, "lr": 8.075499783077321e-06, "episode": 6448, "epoch": 0.32565656565656564, "step": 403}, {"eps": 0, "objective/kl": 20.830650329589844, "objective/entropy": -34.34098434448242, "objective/non_score_reward": -1.0415325164794922, "objective/rlhf_reward": 8.931123733520508, "objective/scores": 10.0, "policy/approxkl_avg": 0.001183489803224802, "policy/clipfrac_avg": 0.011174894869327545, "loss/policy_avg": -0.07570268213748932, "loss/value_avg": 0.5015542507171631, "val/clipfrac_avg": 0.081820048391819, "policy/entropy_avg": 0.18660354614257812, "val/ratio": 0.9876670837402344, "val/ratio_var": 4.6252916945377365e-05, "val/num_eos_tokens": 52, "lr": 8.064957418316014e-06, "episode": 6464, "epoch": 0.32646464646464646, "step": 404}, {"eps": 0, "objective/kl": 23.338232040405273, "objective/entropy": -124.3838119506836, "objective/non_score_reward": -1.1669116020202637, "objective/rlhf_reward": 8.567462921142578, "objective/scores": 9.75, "policy/approxkl_avg": 0.0008951124618761241, "policy/clipfrac_avg": 0.017911501228809357, "loss/policy_avg": 0.2763940095901489, "loss/value_avg": 0.7284906506538391, "val/clipfrac_avg": 0.062357835471630096, "policy/entropy_avg": 0.14433574676513672, "val/ratio": 0.990904688835144, "val/ratio_var": 2.749706618487835e-05, "val/num_eos_tokens": 413, "lr": 8.054393180507572e-06, "episode": 6480, "epoch": 0.32727272727272727, "step": 405}, {"eps": 0, "objective/kl": 23.43341064453125, "objective/entropy": -124.5007095336914, "objective/non_score_reward": -1.1716704368591309, "objective/rlhf_reward": 8.894735336303711, "objective/scores": 10.0625, "policy/approxkl_avg": 0.0008479490061290562, "policy/clipfrac_avg": 0.013162427581846714, "loss/policy_avg": -0.36677244305610657, "loss/value_avg": 0.46131956577301025, "val/clipfrac_avg": 0.1444421261548996, "policy/entropy_avg": 0.19167137145996094, "val/ratio": 0.9900524020195007, "val/ratio_var": 6.206807302078232e-05, "val/num_eos_tokens": 311, "lr": 8.043807145043604e-06, "episode": 6496, "epoch": 0.3280808080808081, "step": 406}, {"eps": 0, "objective/kl": 19.725975036621094, "objective/entropy": -91.82843017578125, "objective/non_score_reward": -0.9862988591194153, "objective/rlhf_reward": 10.459013938903809, "objective/scores": 11.4375, "policy/approxkl_avg": 0.0008222603937610984, "policy/clipfrac_avg": 0.0158871840685606, "loss/policy_avg": 0.04725009202957153, "loss/value_avg": 1.2189075946807861, "val/clipfrac_avg": 0.16988256573677063, "policy/entropy_avg": 0.13180410861968994, "val/ratio": 0.9913500547409058, "val/ratio_var": 4.944593092659488e-05, "val/num_eos_tokens": 286, "lr": 8.033199387471278e-06, "episode": 6512, "epoch": 0.3288888888888889, "step": 407}, {"eps": 0, "objective/kl": 22.473175048828125, "objective/entropy": -143.9917755126953, "objective/non_score_reward": -1.1236587762832642, "objective/rlhf_reward": 10.821653366088867, "objective/scores": 11.9375, "policy/approxkl_avg": 0.0006903298199176788, "policy/clipfrac_avg": 0.02162603661417961, "loss/policy_avg": -0.465353399515152, "loss/value_avg": 0.7926937341690063, "val/clipfrac_avg": 0.14454638957977295, "policy/entropy_avg": 0.16019591689109802, "val/ratio": 0.9919313192367554, "val/ratio_var": 4.090562288183719e-05, "val/num_eos_tokens": 448, "lr": 8.022569983492781e-06, "episode": 6528, "epoch": 0.3296969696969697, "step": 408}, {"eps": 0, "objective/kl": 22.801063537597656, "objective/entropy": -75.08297729492188, "objective/non_score_reward": -1.1400532722473145, "objective/rlhf_reward": 9.375572204589844, "objective/scores": 10.5, "policy/approxkl_avg": 0.0012238363269716501, "policy/clipfrac_avg": 0.013967175967991352, "loss/policy_avg": 0.07690078765153885, "loss/value_avg": 0.29455649852752686, "val/clipfrac_avg": 0.19421738386154175, "policy/entropy_avg": 0.15617752075195312, "val/ratio": 0.9893690347671509, "val/ratio_var": 5.305988452164456e-05, "val/num_eos_tokens": 56, "lr": 8.011919008964779e-06, "episode": 6544, "epoch": 0.3305050505050505, "step": 409}, {"eps": 0, "objective/kl": 23.710628509521484, "objective/entropy": -171.51019287109375, "objective/non_score_reward": -1.1855316162109375, "objective/rlhf_reward": 10.869155883789062, "objective/scores": 12.0625, "policy/approxkl_avg": 0.000929538335185498, "policy/clipfrac_avg": 0.030162684619426727, "loss/policy_avg": 0.10671384632587433, "loss/value_avg": 0.24573181569576263, "val/clipfrac_avg": 0.07078950107097626, "policy/entropy_avg": 0.17933368682861328, "val/ratio": 0.9917181730270386, "val/ratio_var": 3.602935612434521e-05, "val/num_eos_tokens": 479, "lr": 8.001246539897877e-06, "episode": 6560, "epoch": 0.33131313131313134, "step": 410}, {"eps": 0, "objective/kl": 18.754131317138672, "objective/entropy": -92.83210754394531, "objective/non_score_reward": -0.937706470489502, "objective/rlhf_reward": 11.253700256347656, "objective/scores": 12.1875, "policy/approxkl_avg": 0.00083785445895046, "policy/clipfrac_avg": 0.008017129264771938, "loss/policy_avg": -0.28077971935272217, "loss/value_avg": 1.3987998962402344, "val/clipfrac_avg": 0.18445537984371185, "policy/entropy_avg": 0.15318012237548828, "val/ratio": 0.9898405075073242, "val/ratio_var": 7.417797314701602e-05, "val/num_eos_tokens": 23, "lr": 7.99055265245608e-06, "episode": 6576, "epoch": 0.3321212121212121, "step": 411}, {"eps": 0, "objective/kl": 25.964508056640625, "objective/entropy": -66.04624938964844, "objective/non_score_reward": -1.2982254028320312, "objective/rlhf_reward": 9.779899597167969, "objective/scores": 11.0625, "policy/approxkl_avg": 0.0010015174048021436, "policy/clipfrac_avg": 0.00993548333644867, "loss/policy_avg": -0.18175235390663147, "loss/value_avg": 0.18787693977355957, "val/clipfrac_avg": 0.04342540353536606, "policy/entropy_avg": 0.17697501182556152, "val/ratio": 0.9883245825767517, "val/ratio_var": 5.8682868257164955e-05, "val/num_eos_tokens": 47, "lr": 7.979837422956236e-06, "episode": 6592, "epoch": 0.3329292929292929, "step": 412}, {"eps": 0, "objective/kl": 25.512779235839844, "objective/entropy": -34.08283233642578, "objective/non_score_reward": -1.2756389379501343, "objective/rlhf_reward": 9.029048919677734, "objective/scores": 10.3125, "policy/approxkl_avg": 0.001080115558579564, "policy/clipfrac_avg": 0.009306976571679115, "loss/policy_avg": 0.22827568650245667, "loss/value_avg": 0.5228079557418823, "val/clipfrac_avg": 0.04794730246067047, "policy/entropy_avg": 0.20532989501953125, "val/ratio": 0.9874177575111389, "val/ratio_var": 2.4738281354075298e-05, "val/num_eos_tokens": 18, "lr": 7.969100927867508e-06, "episode": 6608, "epoch": 0.3337373737373737, "step": 413}, {"eps": 0, "objective/kl": 17.2120304107666, "objective/entropy": -88.73225402832031, "objective/non_score_reward": -0.860601544380188, "objective/rlhf_reward": 12.131586074829102, "objective/scores": 13.0, "policy/approxkl_avg": 0.0006878376007080078, "policy/clipfrac_avg": 0.005827655084431171, "loss/policy_avg": -0.2480793595314026, "loss/value_avg": 0.7881271243095398, "val/clipfrac_avg": 0.22013108432292938, "policy/entropy_avg": 0.1268022060394287, "val/ratio": 0.9918868541717529, "val/ratio_var": 6.459503492806107e-05, "val/num_eos_tokens": 113, "lr": 7.958343243810818e-06, "episode": 6624, "epoch": 0.33454545454545453, "step": 414}, {"eps": 0, "objective/kl": 22.453895568847656, "objective/entropy": -65.54212951660156, "objective/non_score_reward": -1.1226948499679565, "objective/rlhf_reward": 11.283555030822754, "objective/scores": 12.375, "policy/approxkl_avg": 0.0008567589102312922, "policy/clipfrac_avg": 0.005040471442043781, "loss/policy_avg": -0.007346153259277344, "loss/value_avg": 1.3498705625534058, "val/clipfrac_avg": 0.10788097977638245, "policy/entropy_avg": 0.16246891021728516, "val/ratio": 0.9893018007278442, "val/ratio_var": 4.8482721467735246e-05, "val/num_eos_tokens": 322, "lr": 7.9475644475583e-06, "episode": 6640, "epoch": 0.33535353535353535, "step": 415}, {"eps": 0, "objective/kl": 13.174534797668457, "objective/entropy": -25.829357147216797, "objective/non_score_reward": -0.6587268114089966, "objective/rlhf_reward": 10.313929557800293, "objective/scores": 11.0, "policy/approxkl_avg": 0.0008746474632062018, "policy/clipfrac_avg": 0.006421862170100212, "loss/policy_avg": 0.31426429748535156, "loss/value_avg": 2.21596097946167, "val/clipfrac_avg": 0.15579935908317566, "policy/entropy_avg": 0.19477272033691406, "val/ratio": 0.9900082349777222, "val/ratio_var": 9.775400394573808e-05, "val/num_eos_tokens": 7, "lr": 7.93676461603276e-06, "episode": 6656, "epoch": 0.33616161616161616, "step": 416}, {"eps": 0, "objective/kl": 20.854877471923828, "objective/entropy": -116.59674072265625, "objective/non_score_reward": -1.0427439212799072, "objective/rlhf_reward": 13.402568817138672, "objective/scores": 14.4375, "policy/approxkl_avg": 0.000628868059720844, "policy/clipfrac_avg": 0.003246631007641554, "loss/policy_avg": -0.5780159831047058, "loss/value_avg": 0.7495456337928772, "val/clipfrac_avg": 0.10604649782180786, "policy/entropy_avg": 0.12189757823944092, "val/ratio": 0.9918810129165649, "val/ratio_var": 3.4486944059608504e-05, "val/num_eos_tokens": 124, "lr": 7.925943826307119e-06, "episode": 6672, "epoch": 0.336969696969697, "step": 417}, {"eps": 0, "objective/kl": 25.002452850341797, "objective/entropy": -129.1641082763672, "objective/non_score_reward": -1.2501227855682373, "objective/rlhf_reward": 9.6483154296875, "objective/scores": 10.875, "policy/approxkl_avg": 0.0008685050415806472, "policy/clipfrac_avg": 0.0092330202460289, "loss/policy_avg": 0.12082649767398834, "loss/value_avg": 0.7548036575317383, "val/clipfrac_avg": 0.1482473611831665, "policy/entropy_avg": 0.16512155532836914, "val/ratio": 0.9887162446975708, "val/ratio_var": 7.209738396340981e-05, "val/num_eos_tokens": 431, "lr": 7.91510215560387e-06, "episode": 6688, "epoch": 0.3377777777777778, "step": 418}, {"eps": 0, "objective/kl": 17.616220474243164, "objective/entropy": -99.79671478271484, "objective/non_score_reward": -0.8808110952377319, "objective/rlhf_reward": 10.90825080871582, "objective/scores": 11.8125, "policy/approxkl_avg": 0.0006818549009039998, "policy/clipfrac_avg": 0.010476157069206238, "loss/policy_avg": 0.3398967981338501, "loss/value_avg": 1.4188909530639648, "val/clipfrac_avg": 0.1959417462348938, "policy/entropy_avg": 0.1477426290512085, "val/ratio": 0.9921755790710449, "val/ratio_var": 3.494379416224547e-05, "val/num_eos_tokens": 87, "lr": 7.904239681294515e-06, "episode": 6704, "epoch": 0.3385858585858586, "step": 419}, {"eps": 0, "objective/kl": 20.073959350585938, "objective/entropy": -143.52662658691406, "objective/non_score_reward": -1.0036978721618652, "objective/rlhf_reward": 11.246301651000977, "objective/scores": 12.25, "policy/approxkl_avg": 0.0007999450317583978, "policy/clipfrac_avg": 0.005896029528230429, "loss/policy_avg": -0.20346836745738983, "loss/value_avg": 1.3521277904510498, "val/clipfrac_avg": 0.10764417052268982, "policy/entropy_avg": 0.1470661163330078, "val/ratio": 0.9900174140930176, "val/ratio_var": 5.0762690079864115e-05, "val/num_eos_tokens": 184, "lr": 7.89335648089903e-06, "episode": 6720, "epoch": 0.3393939393939394, "step": 420}, {"eps": 0, "objective/kl": 24.23324203491211, "objective/entropy": -125.03632354736328, "objective/non_score_reward": -1.2116621732711792, "objective/rlhf_reward": 9.202400207519531, "objective/scores": 10.4375, "policy/approxkl_avg": 0.0008304300718009472, "policy/clipfrac_avg": 0.004866223782300949, "loss/policy_avg": 0.3638865351676941, "loss/value_avg": 0.498892605304718, "val/clipfrac_avg": 0.11759978532791138, "policy/entropy_avg": 0.15968847274780273, "val/ratio": 0.9900087118148804, "val/ratio_var": 5.0026934331981465e-05, "val/num_eos_tokens": 266, "lr": 7.882452632085296e-06, "episode": 6736, "epoch": 0.3402020202020202, "step": 421}, {"eps": 0, "objective/kl": 24.247493743896484, "objective/entropy": -25.790729522705078, "objective/non_score_reward": -1.2123748064041138, "objective/rlhf_reward": 10.975125312805176, "objective/scores": 12.1875, "policy/approxkl_avg": 0.0010115428594872355, "policy/clipfrac_avg": 0.006701755337417126, "loss/policy_avg": 0.18302330374717712, "loss/value_avg": 0.675520122051239, "val/clipfrac_avg": 0.14078916609287262, "policy/entropy_avg": 0.16680622100830078, "val/ratio": 0.988139271736145, "val/ratio_var": 6.585121445823461e-05, "val/num_eos_tokens": 30, "lr": 7.871528212668556e-06, "episode": 6752, "epoch": 0.341010101010101, "step": 422}, {"eps": 0, "objective/kl": 23.447364807128906, "objective/entropy": -99.92378234863281, "objective/non_score_reward": -1.1723682880401611, "objective/rlhf_reward": 10.062005996704102, "objective/scores": 11.25, "policy/approxkl_avg": 0.0008515919907949865, "policy/clipfrac_avg": 0.011789527721703053, "loss/policy_avg": 0.1767496019601822, "loss/value_avg": 0.8422822952270508, "val/clipfrac_avg": 0.07063312828540802, "policy/entropy_avg": 0.1661376953125, "val/ratio": 0.9900652766227722, "val/ratio_var": 2.7732188755180687e-05, "val/num_eos_tokens": 82, "lr": 7.860583300610849e-06, "episode": 6768, "epoch": 0.3418181818181818, "step": 423}, {"eps": 0, "objective/kl": 22.923152923583984, "objective/entropy": -87.34598541259766, "objective/non_score_reward": -1.1461577415466309, "objective/rlhf_reward": 10.056966781616211, "objective/scores": 11.1875, "policy/approxkl_avg": 0.001004643039777875, "policy/clipfrac_avg": 0.01210677158087492, "loss/policy_avg": -0.09848126024007797, "loss/value_avg": 0.2839937210083008, "val/clipfrac_avg": 0.06793520599603653, "policy/entropy_avg": 0.15830230712890625, "val/ratio": 0.9890733361244202, "val/ratio_var": 4.017764877062291e-05, "val/num_eos_tokens": 316, "lr": 7.849617974020467e-06, "episode": 6784, "epoch": 0.3426262626262626, "step": 424}, {"eps": 0, "objective/kl": 20.979629516601562, "objective/entropy": -24.3656005859375, "objective/non_score_reward": -1.0489814281463623, "objective/rlhf_reward": 10.318206787109375, "objective/scores": 11.375, "policy/approxkl_avg": 0.0009983977070078254, "policy/clipfrac_avg": 0.006080445367842913, "loss/policy_avg": -0.4337696433067322, "loss/value_avg": 1.939749002456665, "val/clipfrac_avg": 0.10390697419643402, "policy/entropy_avg": 0.1742689609527588, "val/ratio": 0.9901939630508423, "val/ratio_var": 5.642051110044122e-05, "val/num_eos_tokens": 217, "lr": 7.838632311151384e-06, "episode": 6800, "epoch": 0.3434343434343434, "step": 425}, {"eps": 0, "objective/kl": 27.880338668823242, "objective/entropy": -38.45948791503906, "objective/non_score_reward": -1.3940168619155884, "objective/rlhf_reward": 9.11379623413086, "objective/scores": 10.5, "policy/approxkl_avg": 0.00097821862436831, "policy/clipfrac_avg": 0.006474747322499752, "loss/policy_avg": 0.0010461192578077316, "loss/value_avg": 0.22941997647285461, "val/clipfrac_avg": 0.03179371729493141, "policy/entropy_avg": 0.18066978454589844, "val/ratio": 0.9881677627563477, "val/ratio_var": 3.830140849458985e-05, "val/num_eos_tokens": 113, "lr": 7.827626390402707e-06, "episode": 6816, "epoch": 0.34424242424242424, "step": 426}, {"eps": 0, "objective/kl": 30.09839630126953, "objective/entropy": -61.58184051513672, "objective/non_score_reward": -1.5049197673797607, "objective/rlhf_reward": 9.81539249420166, "objective/scores": 11.3125, "policy/approxkl_avg": 0.0010863475035876036, "policy/clipfrac_avg": 0.01059107854962349, "loss/policy_avg": -0.07846693694591522, "loss/value_avg": 0.26400119066238403, "val/clipfrac_avg": 0.037561144679784775, "policy/entropy_avg": 0.15493392944335938, "val/ratio": 0.9897431135177612, "val/ratio_var": 4.0529008401790634e-05, "val/num_eos_tokens": 138, "lr": 7.81660029031811e-06, "episode": 6832, "epoch": 0.34505050505050505, "step": 427}, {"eps": 0, "objective/kl": 21.427749633789062, "objective/entropy": -48.85191345214844, "objective/non_score_reward": -1.0713874101638794, "objective/rlhf_reward": 9.936426162719727, "objective/scores": 11.0, "policy/approxkl_avg": 0.0023591392673552036, "policy/clipfrac_avg": 0.019066959619522095, "loss/policy_avg": -0.38922715187072754, "loss/value_avg": 0.6340113282203674, "val/clipfrac_avg": 0.08839962631464005, "policy/entropy_avg": 0.20233774185180664, "val/ratio": 0.9836812019348145, "val/ratio_var": 0.0002559312852099538, "val/num_eos_tokens": 56, "lr": 7.805554089585284e-06, "episode": 6848, "epoch": 0.34585858585858587, "step": 428}, {"eps": 0, "objective/kl": 19.139610290527344, "objective/entropy": -94.49333190917969, "objective/non_score_reward": -0.9569804668426514, "objective/rlhf_reward": 13.48051929473877, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0010149467270821333, "policy/clipfrac_avg": 0.009838546626269817, "loss/policy_avg": -0.4161893427371979, "loss/value_avg": 1.773421049118042, "val/clipfrac_avg": 0.18258625268936157, "policy/entropy_avg": 0.16881978511810303, "val/ratio": 0.9892622232437134, "val/ratio_var": 8.773741137702018e-05, "val/num_eos_tokens": 84, "lr": 7.794487867035358e-06, "episode": 6864, "epoch": 0.3466666666666667, "step": 429}, {"eps": 0, "objective/kl": 28.28658676147461, "objective/entropy": -144.0445098876953, "objective/non_score_reward": -1.4143294095993042, "objective/rlhf_reward": 11.101295471191406, "objective/scores": 12.5, "policy/approxkl_avg": 0.0007755355327390134, "policy/clipfrac_avg": 0.005785915069282055, "loss/policy_avg": -0.36678922176361084, "loss/value_avg": 0.3908446133136749, "val/clipfrac_avg": 0.11318591237068176, "policy/entropy_avg": 0.15067648887634277, "val/ratio": 0.9902333617210388, "val/ratio_var": 4.714354145107791e-05, "val/num_eos_tokens": 683, "lr": 7.783401701642356e-06, "episode": 6880, "epoch": 0.3474747474747475, "step": 430}, {"eps": 0, "objective/kl": 24.70610809326172, "objective/entropy": -88.11723327636719, "objective/non_score_reward": -1.2353053092956543, "objective/rlhf_reward": 11.608444213867188, "objective/scores": 12.875, "policy/approxkl_avg": 0.0012505800696089864, "policy/clipfrac_avg": 0.017380211502313614, "loss/policy_avg": -0.07360352575778961, "loss/value_avg": 0.4391564130783081, "val/clipfrac_avg": 0.08813799917697906, "policy/entropy_avg": 0.1649489402770996, "val/ratio": 0.9886548519134521, "val/ratio_var": 8.897272346075624e-05, "val/num_eos_tokens": 396, "lr": 7.772295672522615e-06, "episode": 6896, "epoch": 0.3482828282828283, "step": 431}, {"eps": 0, "objective/kl": 25.28207015991211, "objective/entropy": -98.53260803222656, "objective/non_score_reward": -1.2641034126281738, "objective/rlhf_reward": 10.774959564208984, "objective/scores": 12.0625, "policy/approxkl_avg": 0.0009465490002185106, "policy/clipfrac_avg": 0.017566177994012833, "loss/policy_avg": -0.3927380442619324, "loss/value_avg": 0.7407118082046509, "val/clipfrac_avg": 0.05415056645870209, "policy/entropy_avg": 0.17182016372680664, "val/ratio": 0.9887499809265137, "val/ratio_var": 5.970060738036409e-05, "val/num_eos_tokens": 205, "lr": 7.761169858934238e-06, "episode": 6912, "epoch": 0.3490909090909091, "step": 432}, {"eps": 0, "objective/kl": 29.98111915588379, "objective/entropy": -105.76652526855469, "objective/non_score_reward": -1.4990559816360474, "objective/rlhf_reward": 11.250944137573242, "objective/scores": 12.75, "policy/approxkl_avg": 0.0008196796989068389, "policy/clipfrac_avg": 0.014055829495191574, "loss/policy_avg": 0.07162998616695404, "loss/value_avg": 0.6784456968307495, "val/clipfrac_avg": 0.10481823980808258, "policy/entropy_avg": 0.16859722137451172, "val/ratio": 0.9896249771118164, "val/ratio_var": 6.0306902014417574e-05, "val/num_eos_tokens": 508, "lr": 7.750024340276512e-06, "episode": 6928, "epoch": 0.3498989898989899, "step": 433}, {"eps": 0, "objective/kl": 24.68207550048828, "objective/entropy": -118.70050048828125, "objective/non_score_reward": -1.2341036796569824, "objective/rlhf_reward": 10.75808334350586, "objective/scores": 12.0, "policy/approxkl_avg": 0.000771071354392916, "policy/clipfrac_avg": 0.012887731194496155, "loss/policy_avg": 0.21741759777069092, "loss/value_avg": 0.4865216314792633, "val/clipfrac_avg": 0.1991758644580841, "policy/entropy_avg": 0.12623023986816406, "val/ratio": 0.9922160506248474, "val/ratio_var": 3.5436314647085965e-05, "val/num_eos_tokens": 560, "lr": 7.738859196089358e-06, "episode": 6944, "epoch": 0.3507070707070707, "step": 434}, {"eps": 0, "objective/kl": 31.92902374267578, "objective/entropy": -127.72758483886719, "objective/non_score_reward": -1.5964511632919312, "objective/rlhf_reward": 10.106673240661621, "objective/scores": 11.6875, "policy/approxkl_avg": 0.0008912943303585052, "policy/clipfrac_avg": 0.006930951029062271, "loss/policy_avg": -0.07316741347312927, "loss/value_avg": 0.22129662334918976, "val/clipfrac_avg": 0.07383783161640167, "policy/entropy_avg": 0.14632463455200195, "val/ratio": 0.989611029624939, "val/ratio_var": 6.377204408636317e-05, "val/num_eos_tokens": 833, "lr": 7.727674506052744e-06, "episode": 6960, "epoch": 0.3515151515151515, "step": 435}, {"eps": 0, "objective/kl": 32.4725227355957, "objective/entropy": -99.71429443359375, "objective/non_score_reward": -1.6236259937286377, "objective/rlhf_reward": 10.235749244689941, "objective/scores": 11.875, "policy/approxkl_avg": 0.0009519133600406349, "policy/clipfrac_avg": 0.015381633304059505, "loss/policy_avg": 0.27702921628952026, "loss/value_avg": 0.9074195623397827, "val/clipfrac_avg": 0.3032349646091461, "policy/entropy_avg": 0.17629116773605347, "val/ratio": 0.9885252714157104, "val/ratio_var": 7.608420855831355e-05, "val/num_eos_tokens": 165, "lr": 7.716470349986134e-06, "episode": 6976, "epoch": 0.3523232323232323, "step": 436}, {"eps": 0, "objective/kl": 29.33208656311035, "objective/entropy": -67.62409973144531, "objective/non_score_reward": -1.466604471206665, "objective/rlhf_reward": 12.588083267211914, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0009833010844886303, "policy/clipfrac_avg": 0.006308242678642273, "loss/policy_avg": -0.10949157923460007, "loss/value_avg": 0.7171697020530701, "val/clipfrac_avg": 0.22335688769817352, "policy/entropy_avg": 0.14640045166015625, "val/ratio": 0.9893021583557129, "val/ratio_var": 3.676968117360957e-05, "val/num_eos_tokens": 42, "lr": 7.70524680784791e-06, "episode": 6992, "epoch": 0.35313131313131313, "step": 437}, {"eps": 0, "objective/kl": 29.602069854736328, "objective/entropy": -34.55720520019531, "objective/non_score_reward": -1.4801034927368164, "objective/rlhf_reward": 10.379271507263184, "objective/scores": 11.875, "policy/approxkl_avg": 0.001006154459901154, "policy/clipfrac_avg": 0.005651135928928852, "loss/policy_avg": 0.20945397019386292, "loss/value_avg": 0.2541009783744812, "val/clipfrac_avg": 0.1426633596420288, "policy/entropy_avg": 0.17934393882751465, "val/ratio": 0.9886967539787292, "val/ratio_var": 4.757837450597435e-05, "val/num_eos_tokens": 142, "lr": 7.694003959734802e-06, "episode": 7008, "epoch": 0.35393939393939394, "step": 438}, {"eps": 0, "objective/kl": 25.049076080322266, "objective/entropy": -95.60530090332031, "objective/non_score_reward": -1.2524539232254028, "objective/rlhf_reward": 10.74754524230957, "objective/scores": 12.0, "policy/approxkl_avg": 0.0030104375910013914, "policy/clipfrac_avg": 0.009499769657850266, "loss/policy_avg": 0.04139135777950287, "loss/value_avg": 0.8019120097160339, "val/clipfrac_avg": 0.11603914201259613, "policy/entropy_avg": 0.14387786388397217, "val/ratio": 0.9905297160148621, "val/ratio_var": 5.1482784328982234e-05, "val/num_eos_tokens": 222, "lr": 7.682741885881314e-06, "episode": 7024, "epoch": 0.35474747474747476, "step": 439}, {"eps": 0, "objective/kl": 28.195911407470703, "objective/entropy": -84.95793151855469, "objective/non_score_reward": -1.4097956418991089, "objective/rlhf_reward": 10.613641738891602, "objective/scores": 12.0, "policy/approxkl_avg": 0.0007476760656572878, "policy/clipfrac_avg": 0.006465469021350145, "loss/policy_avg": 0.2531706392765045, "loss/value_avg": 0.6514580845832825, "val/clipfrac_avg": 0.1621960699558258, "policy/entropy_avg": 0.12859344482421875, "val/ratio": 0.9909597635269165, "val/ratio_var": 5.390801379689947e-05, "val/num_eos_tokens": 104, "lr": 7.671460666659161e-06, "episode": 7040, "epoch": 0.35555555555555557, "step": 440}, {"eps": 0, "objective/kl": 38.44916915893555, "objective/entropy": -88.81119537353516, "objective/non_score_reward": -1.9224586486816406, "objective/rlhf_reward": 10.08535385131836, "objective/scores": 12.0, "policy/approxkl_avg": 0.000971765024587512, "policy/clipfrac_avg": 0.007417440880089998, "loss/policy_avg": -0.04607268422842026, "loss/value_avg": 0.24199238419532776, "val/clipfrac_avg": 0.10574978590011597, "policy/entropy_avg": 0.18393898010253906, "val/ratio": 0.9876141548156738, "val/ratio_var": 4.917777914670296e-05, "val/num_eos_tokens": 173, "lr": 7.660160382576683e-06, "episode": 7056, "epoch": 0.3563636363636364, "step": 441}, {"eps": 0, "objective/kl": 29.281410217285156, "objective/entropy": -92.62014770507812, "objective/non_score_reward": -1.4640705585479736, "objective/rlhf_reward": 11.551554679870605, "objective/scores": 13.0, "policy/approxkl_avg": 0.0011433872859925032, "policy/clipfrac_avg": 0.012358217500150204, "loss/policy_avg": -0.10014820098876953, "loss/value_avg": 0.7047788500785828, "val/clipfrac_avg": 0.07325705885887146, "policy/entropy_avg": 0.16633868217468262, "val/ratio": 0.9898238778114319, "val/ratio_var": 5.3111554734641686e-05, "val/num_eos_tokens": 292, "lr": 7.648841114278278e-06, "episode": 7072, "epoch": 0.3571717171717172, "step": 442}, {"eps": 0, "objective/kl": 30.107555389404297, "objective/entropy": -49.81909942626953, "objective/non_score_reward": -1.5053777694702148, "objective/rlhf_reward": 9.268058776855469, "objective/scores": 10.75, "policy/approxkl_avg": 0.0012591599952429533, "policy/clipfrac_avg": 0.015652721747756004, "loss/policy_avg": 0.2637108862400055, "loss/value_avg": 0.8885920643806458, "val/clipfrac_avg": 0.06298521906137466, "policy/entropy_avg": 0.16948866844177246, "val/ratio": 0.988282322883606, "val/ratio_var": 3.1702376872999594e-05, "val/num_eos_tokens": 108, "lr": 7.637502942543825e-06, "episode": 7088, "epoch": 0.35797979797979795, "step": 443}, {"eps": 0, "objective/kl": 29.099103927612305, "objective/entropy": -128.95480346679688, "objective/non_score_reward": -1.4549552202224731, "objective/rlhf_reward": 10.170044898986816, "objective/scores": 11.625, "policy/approxkl_avg": 0.0009231308940798044, "policy/clipfrac_avg": 0.007753930985927582, "loss/policy_avg": 0.2935371994972229, "loss/value_avg": 0.7965297698974609, "val/clipfrac_avg": 0.12779459357261658, "policy/entropy_avg": 0.17519092559814453, "val/ratio": 0.9897068738937378, "val/ratio_var": 4.8030484322225675e-05, "val/num_eos_tokens": 712, "lr": 7.626145948288107e-06, "episode": 7104, "epoch": 0.35878787878787877, "step": 444}, {"eps": 0, "objective/kl": 43.964115142822266, "objective/entropy": -141.06298828125, "objective/non_score_reward": -2.1982059478759766, "objective/rlhf_reward": 8.778356552124023, "objective/scores": 11.0, "policy/approxkl_avg": 0.0010534923058003187, "policy/clipfrac_avg": 0.00662950798869133, "loss/policy_avg": -0.10212291777133942, "loss/value_avg": 0.41369742155075073, "val/clipfrac_avg": 0.20534712076187134, "policy/entropy_avg": 0.16738319396972656, "val/ratio": 0.9893473386764526, "val/ratio_var": 6.579797627637163e-05, "val/num_eos_tokens": 35, "lr": 7.614770212560234e-06, "episode": 7120, "epoch": 0.3595959595959596, "step": 445}, {"eps": 0, "objective/kl": 40.433448791503906, "objective/entropy": -137.25640869140625, "objective/non_score_reward": -2.0216727256774902, "objective/rlhf_reward": 9.408015251159668, "objective/scores": 11.4375, "policy/approxkl_avg": 0.0011193909449502826, "policy/clipfrac_avg": 0.009532807394862175, "loss/policy_avg": 0.043000537902116776, "loss/value_avg": 0.2879178524017334, "val/clipfrac_avg": 0.15842124819755554, "policy/entropy_avg": 0.1789846420288086, "val/ratio": 0.9881588816642761, "val/ratio_var": 8.264256757684052e-05, "val/num_eos_tokens": 429, "lr": 7.603375816543065e-06, "episode": 7136, "epoch": 0.3604040404040404, "step": 446}, {"eps": 0, "objective/kl": 39.474517822265625, "objective/entropy": -107.86872863769531, "objective/non_score_reward": -1.9737260341644287, "objective/rlhf_reward": 9.784086227416992, "objective/scores": 11.75, "policy/approxkl_avg": 0.000841513741761446, "policy/clipfrac_avg": 0.006178049370646477, "loss/policy_avg": -0.09768345952033997, "loss/value_avg": 0.20441338419914246, "val/clipfrac_avg": 0.12125139683485031, "policy/entropy_avg": 0.1564655303955078, "val/ratio": 0.9900634288787842, "val/ratio_var": 4.8899408284341916e-05, "val/num_eos_tokens": 35, "lr": 7.591962841552627e-06, "episode": 7152, "epoch": 0.3612121212121212, "step": 447}, {"eps": 0, "objective/kl": 34.39695739746094, "objective/entropy": -65.5897216796875, "objective/non_score_reward": -1.7198479175567627, "objective/rlhf_reward": 10.537964820861816, "objective/scores": 12.25, "policy/approxkl_avg": 0.0009984057396650314, "policy/clipfrac_avg": 0.005852546542882919, "loss/policy_avg": -0.23434095084667206, "loss/value_avg": 0.9878807663917542, "val/clipfrac_avg": 0.03700433671474457, "policy/entropy_avg": 0.17050552368164062, "val/ratio": 0.9879032373428345, "val/ratio_var": 6.289610610110685e-05, "val/num_eos_tokens": 266, "lr": 7.580531369037534e-06, "episode": 7168, "epoch": 0.362020202020202, "step": 448}, {"eps": 0, "objective/kl": 30.470748901367188, "objective/entropy": -42.06928634643555, "objective/non_score_reward": -1.5235373973846436, "objective/rlhf_reward": 10.898337364196777, "objective/scores": 12.4375, "policy/approxkl_avg": 0.0008559878915548325, "policy/clipfrac_avg": 0.008136066608130932, "loss/policy_avg": -0.09545287489891052, "loss/value_avg": 0.35513487458229065, "val/clipfrac_avg": 0.13745607435703278, "policy/entropy_avg": 0.14156532287597656, "val/ratio": 0.9917177557945251, "val/ratio_var": 5.956144741503522e-05, "val/num_eos_tokens": 263, "lr": 7.569081480578413e-06, "episode": 7184, "epoch": 0.36282828282828283, "step": 449}, {"eps": 0, "objective/kl": 36.183135986328125, "objective/entropy": -166.6522674560547, "objective/non_score_reward": -1.8091567754745483, "objective/rlhf_reward": 9.18303108215332, "objective/scores": 11.0, "policy/approxkl_avg": 0.0012210204731673002, "policy/clipfrac_avg": 0.011477088555693626, "loss/policy_avg": -0.09895340353250504, "loss/value_avg": 0.6456364989280701, "val/clipfrac_avg": 0.14100708067417145, "policy/entropy_avg": 0.15171527862548828, "val/ratio": 0.9893430471420288, "val/ratio_var": 6.879774446133524e-05, "val/num_eos_tokens": 196, "lr": 7.55761325788731e-06, "episode": 7200, "epoch": 0.36363636363636365, "step": 450}, {"eps": 0, "objective/kl": 28.169822692871094, "objective/entropy": -95.6431884765625, "objective/non_score_reward": -1.4084912538528442, "objective/rlhf_reward": 10.372758865356445, "objective/scores": 11.75, "policy/approxkl_avg": 0.0009328583255410194, "policy/clipfrac_avg": 0.008236020803451538, "loss/policy_avg": -0.034445568919181824, "loss/value_avg": 1.0610827207565308, "val/clipfrac_avg": 0.06177588552236557, "policy/entropy_avg": 0.1556539535522461, "val/ratio": 0.9893108010292053, "val/ratio_var": 4.2810836021089926e-05, "val/num_eos_tokens": 303, "lr": 7.546126782807117e-06, "episode": 7216, "epoch": 0.36444444444444446, "step": 451}, {"eps": 0, "objective/kl": 26.70556640625, "objective/entropy": -65.11871337890625, "objective/non_score_reward": -1.3352782726287842, "objective/rlhf_reward": 10.086596488952637, "objective/scores": 11.4375, "policy/approxkl_avg": 0.001023412449285388, "policy/clipfrac_avg": 0.013055086135864258, "loss/policy_avg": 0.26358139514923096, "loss/value_avg": 0.5271927118301392, "val/clipfrac_avg": 0.24446935951709747, "policy/entropy_avg": 0.1628122329711914, "val/ratio": 0.9888921976089478, "val/ratio_var": 7.8632976510562e-05, "val/num_eos_tokens": 248, "lr": 7.534622137310983e-06, "episode": 7232, "epoch": 0.3652525252525253, "step": 452}, {"eps": 0, "objective/kl": 23.723403930664062, "objective/entropy": -100.48619079589844, "objective/non_score_reward": -1.186170220375061, "objective/rlhf_reward": 12.63414192199707, "objective/scores": 13.8125, "policy/approxkl_avg": 0.0010372435208410025, "policy/clipfrac_avg": 0.009655164554715157, "loss/policy_avg": -0.20332151651382446, "loss/value_avg": 2.0998940467834473, "val/clipfrac_avg": 0.28355222940444946, "policy/entropy_avg": 0.16978693008422852, "val/ratio": 0.9885410070419312, "val/ratio_var": 5.881414836039767e-05, "val/num_eos_tokens": 168, "lr": 7.52309940350173e-06, "episode": 7248, "epoch": 0.3660606060606061, "step": 453}, {"eps": 0, "objective/kl": 27.353843688964844, "objective/entropy": -121.33277130126953, "objective/non_score_reward": -1.367692232131958, "objective/rlhf_reward": 12.569808006286621, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0009084678022190928, "policy/clipfrac_avg": 0.010569443926215172, "loss/policy_avg": -0.20516440272331238, "loss/value_avg": 0.5702744722366333, "val/clipfrac_avg": 0.10168075561523438, "policy/entropy_avg": 0.1651425063610077, "val/ratio": 0.9904431104660034, "val/ratio_var": 4.341174280853011e-05, "val/num_eos_tokens": 240, "lr": 7.511558663611265e-06, "episode": 7264, "epoch": 0.36686868686868684, "step": 454}, {"eps": 0, "objective/kl": 27.129352569580078, "objective/entropy": -94.4637222290039, "objective/non_score_reward": -1.356467604637146, "objective/rlhf_reward": 10.256814002990723, "objective/scores": 11.625, "policy/approxkl_avg": 0.0010198624804615974, "policy/clipfrac_avg": 0.00805464293807745, "loss/policy_avg": -0.11988245695829391, "loss/value_avg": 0.24273253977298737, "val/clipfrac_avg": 0.05403827503323555, "policy/entropy_avg": 0.1758131980895996, "val/ratio": 0.9884688854217529, "val/ratio_var": 6.81005185469985e-05, "val/num_eos_tokens": 216, "lr": 7.500000000000001e-06, "episode": 7280, "epoch": 0.36767676767676766, "step": 455}, {"eps": 0, "objective/kl": 30.76184844970703, "objective/entropy": -71.29700469970703, "objective/non_score_reward": -1.5380924940109253, "objective/rlhf_reward": 11.415032386779785, "objective/scores": 12.9375, "policy/approxkl_avg": 0.0010650574695318937, "policy/clipfrac_avg": 0.013442248106002808, "loss/policy_avg": 0.15036238729953766, "loss/value_avg": 1.3548328876495361, "val/clipfrac_avg": 0.18505151569843292, "policy/entropy_avg": 0.1958608627319336, "val/ratio": 0.9862017631530762, "val/ratio_var": 8.930258627515286e-05, "val/num_eos_tokens": 102, "lr": 7.488423495156258e-06, "episode": 7296, "epoch": 0.36848484848484847, "step": 456}, {"eps": 0, "objective/kl": 24.803741455078125, "objective/entropy": -110.99166870117188, "objective/non_score_reward": -1.240187168121338, "objective/rlhf_reward": 12.86137580871582, "objective/scores": 14.125, "policy/approxkl_avg": 0.0009097999427467585, "policy/clipfrac_avg": 0.014383969828486443, "loss/policy_avg": -0.03644943982362747, "loss/value_avg": 0.6813304424285889, "val/clipfrac_avg": 0.1426132172346115, "policy/entropy_avg": 0.14606904983520508, "val/ratio": 0.9902492165565491, "val/ratio_var": 5.0029055273626e-05, "val/num_eos_tokens": 396, "lr": 7.47682923169568e-06, "episode": 7312, "epoch": 0.3692929292929293, "step": 457}, {"eps": 0, "objective/kl": 22.956039428710938, "objective/entropy": -49.973045349121094, "objective/non_score_reward": -1.1478021144866943, "objective/rlhf_reward": 10.875635147094727, "objective/scores": 12.0, "policy/approxkl_avg": 0.0010175809729844332, "policy/clipfrac_avg": 0.011250700801610947, "loss/policy_avg": -0.08388765901327133, "loss/value_avg": 0.9905740022659302, "val/clipfrac_avg": 0.21617577970027924, "policy/entropy_avg": 0.20994114875793457, "val/ratio": 0.9885356426239014, "val/ratio_var": 7.953841850394383e-05, "val/num_eos_tokens": 59, "lr": 7.4652172923606494e-06, "episode": 7328, "epoch": 0.3701010101010101, "step": 458}, {"eps": 0, "objective/kl": 27.69005012512207, "objective/entropy": -133.6717529296875, "objective/non_score_reward": -1.3845025300979614, "objective/rlhf_reward": 10.365497589111328, "objective/scores": 11.75, "policy/approxkl_avg": 0.0008332305587828159, "policy/clipfrac_avg": 0.01680421084165573, "loss/policy_avg": 0.12381671369075775, "loss/value_avg": 0.41641736030578613, "val/clipfrac_avg": 0.23695185780525208, "policy/entropy_avg": 0.13964557647705078, "val/ratio": 0.9924179315567017, "val/ratio_var": 7.448666292475536e-05, "val/num_eos_tokens": 526, "lr": 7.453587760019691e-06, "episode": 7344, "epoch": 0.3709090909090909, "step": 459}, {"eps": 0, "objective/kl": 25.1827392578125, "objective/entropy": -37.12174987792969, "objective/non_score_reward": -1.2591369152069092, "objective/rlhf_reward": 12.850236892700195, "objective/scores": 14.125, "policy/approxkl_avg": 0.0010175569914281368, "policy/clipfrac_avg": 0.010180214419960976, "loss/policy_avg": -0.17677900195121765, "loss/value_avg": 2.0295965671539307, "val/clipfrac_avg": 0.2530834674835205, "policy/entropy_avg": 0.1750507354736328, "val/ratio": 0.9883590936660767, "val/ratio_var": 5.4351363360183313e-05, "val/num_eos_tokens": 219, "lr": 7.441940717666877e-06, "episode": 7360, "epoch": 0.3717171717171717, "step": 460}, {"eps": 0, "objective/kl": 26.147159576416016, "objective/entropy": -70.19863891601562, "objective/non_score_reward": -1.307357907295227, "objective/rlhf_reward": 11.684829711914062, "objective/scores": 13.0, "policy/approxkl_avg": 0.0010134675540030003, "policy/clipfrac_avg": 0.018840627744793892, "loss/policy_avg": 0.3645497262477875, "loss/value_avg": 0.820085346698761, "val/clipfrac_avg": 0.1551748812198639, "policy/entropy_avg": 0.17819499969482422, "val/ratio": 0.9893249869346619, "val/ratio_var": 4.883955989498645e-05, "val/num_eos_tokens": 67, "lr": 7.4302762484212465e-06, "episode": 7376, "epoch": 0.37252525252525254, "step": 461}, {"eps": 0, "objective/kl": 23.205120086669922, "objective/entropy": -105.91459655761719, "objective/non_score_reward": -1.160256028175354, "objective/rlhf_reward": 11.042869567871094, "objective/scores": 12.1875, "policy/approxkl_avg": 0.0008878573426045477, "policy/clipfrac_avg": 0.0075564016588032246, "loss/policy_avg": 0.3121355473995209, "loss/value_avg": 0.5492252111434937, "val/clipfrac_avg": 0.09090471267700195, "policy/entropy_avg": 0.16464996337890625, "val/ratio": 0.9889609813690186, "val/ratio_var": 4.3533022108022124e-05, "val/num_eos_tokens": 19, "lr": 7.4185944355261996e-06, "episode": 7392, "epoch": 0.37333333333333335, "step": 462}, {"eps": 0, "objective/kl": 24.100055694580078, "objective/entropy": -67.8624267578125, "objective/non_score_reward": -1.2050029039382935, "objective/rlhf_reward": 12.65437126159668, "objective/scores": 13.875, "policy/approxkl_avg": 0.0009261409286409616, "policy/clipfrac_avg": 0.01092602964490652, "loss/policy_avg": -0.22459957003593445, "loss/value_avg": 0.6871347427368164, "val/clipfrac_avg": 0.12751662731170654, "policy/entropy_avg": 0.17773854732513428, "val/ratio": 0.9884368181228638, "val/ratio_var": 7.627475133631378e-05, "val/num_eos_tokens": 290, "lr": 7.406895362348916e-06, "episode": 7408, "epoch": 0.37414141414141416, "step": 463}, {"eps": 0, "objective/kl": 25.083354949951172, "objective/entropy": -128.5548095703125, "objective/non_score_reward": -1.2541677951812744, "objective/rlhf_reward": 10.964582443237305, "objective/scores": 12.25, "policy/approxkl_avg": 0.0009050636435858905, "policy/clipfrac_avg": 0.011509673669934273, "loss/policy_avg": 0.10064420849084854, "loss/value_avg": 0.42092299461364746, "val/clipfrac_avg": 0.08698742836713791, "policy/entropy_avg": 0.1595158576965332, "val/ratio": 0.9895754456520081, "val/ratio_var": 0.00011247485235799104, "val/num_eos_tokens": 233, "lr": 7.395179112379746e-06, "episode": 7424, "epoch": 0.374949494949495, "step": 464}, {"eps": 0, "objective/kl": 23.573081970214844, "objective/entropy": -212.094482421875, "objective/non_score_reward": -1.1786541938781738, "objective/rlhf_reward": 11.715876579284668, "objective/scores": 12.875, "policy/approxkl_avg": 0.0007067503756843507, "policy/clipfrac_avg": 0.011690440587699413, "loss/policy_avg": 0.18652041256427765, "loss/value_avg": 0.6958376169204712, "val/clipfrac_avg": 0.27865731716156006, "policy/entropy_avg": 0.16191935539245605, "val/ratio": 0.991563081741333, "val/ratio_var": 4.854411963606253e-05, "val/num_eos_tokens": 2015, "lr": 7.383445769231628e-06, "episode": 7440, "epoch": 0.37575757575757573, "step": 465}, {"eps": 0, "objective/kl": 32.347415924072266, "objective/entropy": -52.99230194091797, "objective/non_score_reward": -1.6173707246780396, "objective/rlhf_reward": 11.49981689453125, "objective/scores": 13.125, "policy/approxkl_avg": 0.0010288862977176905, "policy/clipfrac_avg": 0.004554249346256256, "loss/policy_avg": -0.026313835754990578, "loss/value_avg": 0.3103347420692444, "val/clipfrac_avg": 0.03159730136394501, "policy/entropy_avg": 0.2293071746826172, "val/ratio": 0.9858705997467041, "val/ratio_var": 7.237601676024497e-05, "val/num_eos_tokens": 52, "lr": 7.371695416639481e-06, "episode": 7456, "epoch": 0.37656565656565655, "step": 466}, {"eps": 0, "objective/kl": 24.184755325317383, "objective/entropy": -167.1389617919922, "objective/non_score_reward": -1.2092376947402954, "objective/rlhf_reward": 11.716543197631836, "objective/scores": 12.9375, "policy/approxkl_avg": 0.0008187098428606987, "policy/clipfrac_avg": 0.012958062812685966, "loss/policy_avg": -0.1737292855978012, "loss/value_avg": 0.4936577081680298, "val/clipfrac_avg": 0.12412194907665253, "policy/entropy_avg": 0.2136235237121582, "val/ratio": 0.9899157881736755, "val/ratio_var": 5.196731217438355e-05, "val/num_eos_tokens": 789, "lr": 7.359928138459615e-06, "episode": 7472, "epoch": 0.37737373737373736, "step": 467}, {"eps": 0, "objective/kl": 26.607398986816406, "objective/entropy": -99.32038879394531, "objective/non_score_reward": -1.3303699493408203, "objective/rlhf_reward": 10.39619255065918, "objective/scores": 11.75, "policy/approxkl_avg": 0.0009252649615518749, "policy/clipfrac_avg": 0.005941381677985191, "loss/policy_avg": -0.2519969046115875, "loss/value_avg": 0.6479045152664185, "val/clipfrac_avg": 0.08323413878679276, "policy/entropy_avg": 0.18393898010253906, "val/ratio": 0.9880777597427368, "val/ratio_var": 0.0001012428620015271, "val/num_eos_tokens": 578, "lr": 7.348144018669129e-06, "episode": 7488, "epoch": 0.3781818181818182, "step": 468}, {"eps": 0, "objective/kl": 28.445619583129883, "objective/entropy": -113.41480255126953, "objective/non_score_reward": -1.4222809076309204, "objective/rlhf_reward": 10.585531234741211, "objective/scores": 12.0, "policy/approxkl_avg": 0.0010699313133955002, "policy/clipfrac_avg": 0.021148769184947014, "loss/policy_avg": 0.11465664207935333, "loss/value_avg": 0.2218267023563385, "val/clipfrac_avg": 0.05555972456932068, "policy/entropy_avg": 0.1771717071533203, "val/ratio": 0.9880005121231079, "val/ratio_var": 5.9720714489230886e-05, "val/num_eos_tokens": 301, "lr": 7.336343141365311e-06, "episode": 7504, "epoch": 0.378989898989899, "step": 469}, {"eps": 0, "objective/kl": 24.17656898498535, "objective/entropy": -66.94224548339844, "objective/non_score_reward": -1.2088284492492676, "objective/rlhf_reward": 12.02554702758789, "objective/scores": 13.25, "policy/approxkl_avg": 0.000874909688718617, "policy/clipfrac_avg": 0.007139209192246199, "loss/policy_avg": -0.16887971758842468, "loss/value_avg": 1.1260387897491455, "val/clipfrac_avg": 0.13150881230831146, "policy/entropy_avg": 0.18168145418167114, "val/ratio": 0.989689290523529, "val/ratio_var": 5.2844137826468796e-05, "val/num_eos_tokens": 267, "lr": 7.324525590765037e-06, "episode": 7520, "epoch": 0.3797979797979798, "step": 470}, {"eps": 0, "objective/kl": 17.821823120117188, "objective/entropy": -28.33433723449707, "objective/non_score_reward": -0.8910911083221436, "objective/rlhf_reward": 10.921408653259277, "objective/scores": 11.8125, "policy/approxkl_avg": 0.0010049407137557864, "policy/clipfrac_avg": 0.0061156535521149635, "loss/policy_avg": -0.1819142997264862, "loss/value_avg": 0.7196761965751648, "val/clipfrac_avg": 0.18608367443084717, "policy/entropy_avg": 0.18752670288085938, "val/ratio": 0.9881096482276917, "val/ratio_var": 4.828600867767818e-05, "val/num_eos_tokens": 99, "lr": 7.312691451204178e-06, "episode": 7536, "epoch": 0.3806060606060606, "step": 471}, {"eps": 0, "objective/kl": 29.089664459228516, "objective/entropy": -104.83464050292969, "objective/non_score_reward": -1.454483151435852, "objective/rlhf_reward": 10.115829467773438, "objective/scores": 11.5625, "policy/approxkl_avg": 0.0008825187105685472, "policy/clipfrac_avg": 0.01230642944574356, "loss/policy_avg": -0.12739495933055878, "loss/value_avg": 0.6806440949440002, "val/clipfrac_avg": 0.1584380865097046, "policy/entropy_avg": 0.16717922687530518, "val/ratio": 0.989440381526947, "val/ratio_var": 6.354587821988389e-05, "val/num_eos_tokens": 160, "lr": 7.300840807136983e-06, "episode": 7552, "epoch": 0.3814141414141414, "step": 472}, {"eps": 0, "objective/kl": 27.92177963256836, "objective/entropy": -16.53008270263672, "objective/non_score_reward": -1.39608895778656, "objective/rlhf_reward": 11.994535446166992, "objective/scores": 13.375, "policy/approxkl_avg": 0.0012045095209032297, "policy/clipfrac_avg": 0.009494627825915813, "loss/policy_avg": 0.013141915202140808, "loss/value_avg": 1.9222874641418457, "val/clipfrac_avg": 0.10058106482028961, "policy/entropy_avg": 0.22336864471435547, "val/ratio": 0.985871434211731, "val/ratio_var": 6.286393181653693e-05, "val/num_eos_tokens": 123, "lr": 7.288973743135495e-06, "episode": 7568, "epoch": 0.38222222222222224, "step": 473}, {"eps": 0, "objective/kl": 33.50300979614258, "objective/entropy": -98.81828308105469, "objective/non_score_reward": -1.6751505136489868, "objective/rlhf_reward": 11.567037582397461, "objective/scores": 13.25, "policy/approxkl_avg": 0.000829789787530899, "policy/clipfrac_avg": 0.008850395679473877, "loss/policy_avg": -0.28272491693496704, "loss/value_avg": 0.5360223054885864, "val/clipfrac_avg": 0.19075022637844086, "policy/entropy_avg": 0.1538858413696289, "val/ratio": 0.9900673031806946, "val/ratio_var": 5.3723222663393244e-05, "val/num_eos_tokens": 412, "lr": 7.277090343888931e-06, "episode": 7584, "epoch": 0.38303030303030305, "step": 474}, {"eps": 0, "objective/kl": 32.43540573120117, "objective/entropy": -70.26910400390625, "objective/non_score_reward": -1.6217703819274902, "objective/rlhf_reward": 11.596979141235352, "objective/scores": 13.25, "policy/approxkl_avg": 0.0009185326634906232, "policy/clipfrac_avg": 0.013037210330367088, "loss/policy_avg": -0.2495127022266388, "loss/value_avg": 0.3245130181312561, "val/clipfrac_avg": 0.0755743458867073, "policy/entropy_avg": 0.1732182502746582, "val/ratio": 0.9902758598327637, "val/ratio_var": 6.952504918444902e-05, "val/num_eos_tokens": 391, "lr": 7.265190694203086e-06, "episode": 7600, "epoch": 0.3838383838383838, "step": 475}, {"eps": 0, "objective/kl": 33.66416931152344, "objective/entropy": -75.98827362060547, "objective/non_score_reward": -1.6832083463668823, "objective/rlhf_reward": 10.512104034423828, "objective/scores": 12.1875, "policy/approxkl_avg": 0.0009120008908212185, "policy/clipfrac_avg": 0.007840285077691078, "loss/policy_avg": -9.876303374767303e-05, "loss/value_avg": 0.577019453048706, "val/clipfrac_avg": 0.17109884321689606, "policy/entropy_avg": 0.14813995361328125, "val/ratio": 0.9897706508636475, "val/ratio_var": 8.284224895760417e-05, "val/num_eos_tokens": 441, "lr": 7.253274878999728e-06, "episode": 7616, "epoch": 0.3846464646464646, "step": 476}, {"eps": 0, "objective/kl": 30.798736572265625, "objective/entropy": -107.81949615478516, "objective/non_score_reward": -1.539936900138855, "objective/rlhf_reward": 10.460063934326172, "objective/scores": 12.0, "policy/approxkl_avg": 0.0008913679048418999, "policy/clipfrac_avg": 0.014285984449088573, "loss/policy_avg": -0.05202428251504898, "loss/value_avg": 0.769780158996582, "val/clipfrac_avg": 0.13049724698066711, "policy/entropy_avg": 0.14296913146972656, "val/ratio": 0.9915003180503845, "val/ratio_var": 3.443012246862054e-05, "val/num_eos_tokens": 247, "lr": 7.241342983315985e-06, "episode": 7632, "epoch": 0.38545454545454544, "step": 477}, {"eps": 0, "objective/kl": 19.90038299560547, "objective/entropy": -69.23106384277344, "objective/non_score_reward": -0.9950191974639893, "objective/rlhf_reward": 13.98935604095459, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008611504454165697, "policy/clipfrac_avg": 0.008371494710445404, "loss/policy_avg": -0.12627719342708588, "loss/value_avg": 0.489132285118103, "val/clipfrac_avg": 0.08708248287439346, "policy/entropy_avg": 0.1330699920654297, "val/ratio": 0.9916375875473022, "val/ratio_var": 4.943081512465142e-05, "val/num_eos_tokens": 13, "lr": 7.229395092303753e-06, "episode": 7648, "epoch": 0.38626262626262625, "step": 478}, {"eps": 0, "objective/kl": 26.035850524902344, "objective/entropy": -116.11354064941406, "objective/non_score_reward": -1.3017925024032593, "objective/rlhf_reward": 10.33102035522461, "objective/scores": 11.625, "policy/approxkl_avg": 0.0008872399921528995, "policy/clipfrac_avg": 0.012195098213851452, "loss/policy_avg": 0.13281303644180298, "loss/value_avg": 1.582331657409668, "val/clipfrac_avg": 0.13728481531143188, "policy/entropy_avg": 0.1528162956237793, "val/ratio": 0.9894596338272095, "val/ratio_var": 9.663304081186652e-05, "val/num_eos_tokens": 93, "lr": 7.217431291229068e-06, "episode": 7664, "epoch": 0.38707070707070707, "step": 479}, {"eps": 0, "objective/kl": 28.44430160522461, "objective/entropy": -102.0567626953125, "objective/non_score_reward": -1.422215223312378, "objective/rlhf_reward": 10.90591049194336, "objective/scores": 12.3125, "policy/approxkl_avg": 0.0007429048418998718, "policy/clipfrac_avg": 0.013889473862946033, "loss/policy_avg": 1.0533212423324585, "loss/value_avg": 1.3323712348937988, "val/clipfrac_avg": 0.10630576312541962, "policy/entropy_avg": 0.16810226440429688, "val/ratio": 0.9902744293212891, "val/ratio_var": 4.4153079215902835e-05, "val/num_eos_tokens": 276, "lr": 7.205451665471515e-06, "episode": 7680, "epoch": 0.3878787878787879, "step": 480}, {"eps": 0, "objective/kl": 34.4516716003418, "objective/entropy": -135.23007202148438, "objective/non_score_reward": -1.7225836515426636, "objective/rlhf_reward": 10.222728729248047, "objective/scores": 11.9375, "policy/approxkl_avg": 0.0011850697919726372, "policy/clipfrac_avg": 0.011513225734233856, "loss/policy_avg": -0.07577499747276306, "loss/value_avg": 0.18845658004283905, "val/clipfrac_avg": 0.017498323693871498, "policy/entropy_avg": 0.1882985234260559, "val/ratio": 0.9870139360427856, "val/ratio_var": 6.371740892063826e-05, "val/num_eos_tokens": 1018, "lr": 7.193456300523607e-06, "episode": 7696, "epoch": 0.3886868686868687, "step": 481}, {"eps": 0, "objective/kl": 30.676597595214844, "objective/entropy": -64.48172760009766, "objective/non_score_reward": -1.533829927444458, "objective/rlhf_reward": 10.427107810974121, "objective/scores": 11.9375, "policy/approxkl_avg": 0.0008949321345426142, "policy/clipfrac_avg": 0.010553957894444466, "loss/policy_avg": 0.030544549226760864, "loss/value_avg": 0.3417617380619049, "val/clipfrac_avg": 0.055152617394924164, "policy/entropy_avg": 0.17522239685058594, "val/ratio": 0.9891567230224609, "val/ratio_var": 4.994387927581556e-05, "val/num_eos_tokens": 151, "lr": 7.181445281990182e-06, "episode": 7712, "epoch": 0.3894949494949495, "step": 482}, {"eps": 0, "objective/kl": 37.32021713256836, "objective/entropy": -52.11445617675781, "objective/non_score_reward": -1.8660109043121338, "objective/rlhf_reward": 10.141801834106445, "objective/scores": 12.0, "policy/approxkl_avg": 0.0009162778733298182, "policy/clipfrac_avg": 0.0067903571762144566, "loss/policy_avg": -0.09279122203588486, "loss/value_avg": 0.98249751329422, "val/clipfrac_avg": 0.06959788501262665, "policy/entropy_avg": 0.19576644897460938, "val/ratio": 0.9883674383163452, "val/ratio_var": 5.329724444891326e-05, "val/num_eos_tokens": 63, "lr": 7.169418695587791e-06, "episode": 7728, "epoch": 0.3903030303030303, "step": 483}, {"eps": 0, "objective/kl": 40.677001953125, "objective/entropy": -79.94268798828125, "objective/non_score_reward": -2.0338501930236816, "objective/rlhf_reward": 11.005212783813477, "objective/scores": 13.0625, "policy/approxkl_avg": 0.0009370314655825496, "policy/clipfrac_avg": 0.00797116756439209, "loss/policy_avg": -0.170138418674469, "loss/value_avg": 0.3470100164413452, "val/clipfrac_avg": 0.05036664009094238, "policy/entropy_avg": 0.17804527282714844, "val/ratio": 0.9882063865661621, "val/ratio_var": 3.965401992900297e-05, "val/num_eos_tokens": 591, "lr": 7.157376627144084e-06, "episode": 7744, "epoch": 0.39111111111111113, "step": 484}, {"eps": 0, "objective/kl": 33.60002899169922, "objective/entropy": -111.34992218017578, "objective/non_score_reward": -1.6800014972686768, "objective/rlhf_reward": 9.390310287475586, "objective/scores": 11.0625, "policy/approxkl_avg": 0.0007860509213060141, "policy/clipfrac_avg": 0.0074859317392110825, "loss/policy_avg": 0.08009683340787888, "loss/value_avg": 0.8802474737167358, "val/clipfrac_avg": 0.1623341590166092, "policy/entropy_avg": 0.15513873100280762, "val/ratio": 0.9912385940551758, "val/ratio_var": 4.484504825086333e-05, "val/num_eos_tokens": 219, "lr": 7.145319162597196e-06, "episode": 7760, "epoch": 0.39191919191919194, "step": 485}, {"eps": 0, "objective/kl": 34.018707275390625, "objective/entropy": -130.91871643066406, "objective/non_score_reward": -1.7009353637695312, "objective/rlhf_reward": 10.592033386230469, "objective/scores": 12.3125, "policy/approxkl_avg": 0.0007868118118494749, "policy/clipfrac_avg": 0.011621111072599888, "loss/policy_avg": 0.31785762310028076, "loss/value_avg": 0.5920059680938721, "val/clipfrac_avg": 0.08654516935348511, "policy/entropy_avg": 0.1484971046447754, "val/ratio": 0.9907491207122803, "val/ratio_var": 4.03225640184246e-05, "val/num_eos_tokens": 387, "lr": 7.1332463879951404e-06, "episode": 7776, "epoch": 0.3927272727272727, "step": 486}, {"eps": 0, "objective/kl": 30.16710090637207, "objective/entropy": -101.36183166503906, "objective/non_score_reward": -1.5083551406860352, "objective/rlhf_reward": 10.515082359313965, "objective/scores": 12.0, "policy/approxkl_avg": 0.0011175391264259815, "policy/clipfrac_avg": 0.02089299075305462, "loss/policy_avg": 0.5671185255050659, "loss/value_avg": 0.6217664480209351, "val/clipfrac_avg": 0.11080241203308105, "policy/entropy_avg": 0.17551612854003906, "val/ratio": 0.9885724186897278, "val/ratio_var": 5.202844113227911e-05, "val/num_eos_tokens": 686, "lr": 7.121158389495187e-06, "episode": 7792, "epoch": 0.3935353535353535, "step": 487}, {"eps": 0, "objective/kl": 34.61153793334961, "objective/entropy": -156.5941925048828, "objective/non_score_reward": -1.7305768728256226, "objective/rlhf_reward": 10.26161003112793, "objective/scores": 12.0, "policy/approxkl_avg": 0.0011252043768763542, "policy/clipfrac_avg": 0.006505176424980164, "loss/policy_avg": -0.12260816991329193, "loss/value_avg": 0.37460604310035706, "val/clipfrac_avg": 0.07414089143276215, "policy/entropy_avg": 0.21551895141601562, "val/ratio": 0.9872661828994751, "val/ratio_var": 7.018823816906661e-05, "val/num_eos_tokens": 29, "lr": 7.109055253363253e-06, "episode": 7808, "epoch": 0.39434343434343433, "step": 488}, {"eps": 0, "objective/kl": 35.93262481689453, "objective/entropy": -90.52662658691406, "objective/non_score_reward": -1.796631097793579, "objective/rlhf_reward": 9.008056640625, "objective/scores": 10.8125, "policy/approxkl_avg": 0.00089329166803509, "policy/clipfrac_avg": 0.007040734402835369, "loss/policy_avg": -0.03806200996041298, "loss/value_avg": 0.5151583552360535, "val/clipfrac_avg": 0.09701868891716003, "policy/entropy_avg": 0.15955352783203125, "val/ratio": 0.990253210067749, "val/ratio_var": 5.959846021141857e-05, "val/num_eos_tokens": 121, "lr": 7.096937065973285e-06, "episode": 7824, "epoch": 0.39515151515151514, "step": 489}, {"eps": 0, "objective/kl": 35.315399169921875, "objective/entropy": -107.76602172851562, "objective/non_score_reward": -1.7657699584960938, "objective/rlhf_reward": 9.812355041503906, "objective/scores": 11.5625, "policy/approxkl_avg": 0.0009423610754311085, "policy/clipfrac_avg": 0.011315671727061272, "loss/policy_avg": 0.18260058760643005, "loss/value_avg": 1.1594921350479126, "val/clipfrac_avg": 0.21639129519462585, "policy/entropy_avg": 0.15435028076171875, "val/ratio": 0.9888960123062134, "val/ratio_var": 6.884950562380254e-05, "val/num_eos_tokens": 486, "lr": 7.084803913806642e-06, "episode": 7840, "epoch": 0.39595959595959596, "step": 490}, {"eps": 0, "objective/kl": 33.98522186279297, "objective/entropy": -41.985530853271484, "objective/non_score_reward": -1.6992610692977905, "objective/rlhf_reward": 12.589801788330078, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0010092597221955657, "policy/clipfrac_avg": 0.01904710754752159, "loss/policy_avg": -0.09957186132669449, "loss/value_avg": 0.7260873317718506, "val/clipfrac_avg": 0.15300215780735016, "policy/entropy_avg": 0.15885639190673828, "val/ratio": 0.9895732402801514, "val/ratio_var": 4.7087891289265826e-05, "val/num_eos_tokens": 182, "lr": 7.072655883451478e-06, "episode": 7856, "epoch": 0.39676767676767677, "step": 491}, {"eps": 0, "objective/kl": 24.98674964904785, "objective/entropy": -34.535614013671875, "objective/non_score_reward": -1.2493375539779663, "objective/rlhf_reward": 11.360037803649902, "objective/scores": 12.625, "policy/approxkl_avg": 0.001240122364833951, "policy/clipfrac_avg": 0.014498218894004822, "loss/policy_avg": -0.07856579124927521, "loss/value_avg": 0.4049687385559082, "val/clipfrac_avg": 0.07531505078077316, "policy/entropy_avg": 0.1824660301208496, "val/ratio": 0.9880272150039673, "val/ratio_var": 9.475303522776812e-05, "val/num_eos_tokens": 198, "lr": 7.060493061602128e-06, "episode": 7872, "epoch": 0.3975757575757576, "step": 492}, {"eps": 0, "objective/kl": 37.4053955078125, "objective/entropy": -74.60337829589844, "objective/non_score_reward": -1.870269775390625, "objective/rlhf_reward": 11.590667724609375, "objective/scores": 13.4375, "policy/approxkl_avg": 0.0008902495028451085, "policy/clipfrac_avg": 0.008073626086115837, "loss/policy_avg": 0.06024222820997238, "loss/value_avg": 0.33333200216293335, "val/clipfrac_avg": 0.055267032235860825, "policy/entropy_avg": 0.16165542602539062, "val/ratio": 0.9903206825256348, "val/ratio_var": 3.266169369453564e-05, "val/num_eos_tokens": 447, "lr": 7.048315535058485e-06, "episode": 7888, "epoch": 0.3983838383838384, "step": 493}, {"eps": 0, "objective/kl": 37.19384765625, "objective/entropy": -54.44144821166992, "objective/non_score_reward": -1.8596923351287842, "objective/rlhf_reward": 14.319994926452637, "objective/scores": 16.125, "policy/approxkl_avg": 0.0012663885718211532, "policy/clipfrac_avg": 0.010278100147843361, "loss/policy_avg": -0.06525195389986038, "loss/value_avg": 0.6768864393234253, "val/clipfrac_avg": 0.3375101685523987, "policy/entropy_avg": 0.19942545890808105, "val/ratio": 0.9859856367111206, "val/ratio_var": 8.737371535971761e-05, "val/num_eos_tokens": 5, "lr": 7.036123390725381e-06, "episode": 7904, "epoch": 0.3991919191919192, "step": 494}, {"eps": 0, "objective/kl": 34.13693618774414, "objective/entropy": -161.6277618408203, "objective/non_score_reward": -1.7068469524383545, "objective/rlhf_reward": 11.644716262817383, "objective/scores": 13.375, "policy/approxkl_avg": 0.0014467749278992414, "policy/clipfrac_avg": 0.01142089907079935, "loss/policy_avg": -0.15978729724884033, "loss/value_avg": 0.4220675230026245, "val/clipfrac_avg": 0.07222695648670197, "policy/entropy_avg": 0.1384185552597046, "val/ratio": 0.989600419998169, "val/ratio_var": 0.00014253522385843098, "val/num_eos_tokens": 325, "lr": 7.023916715611969e-06, "episode": 7920, "epoch": 0.4, "step": 495}, {"eps": 0, "objective/kl": 33.550636291503906, "objective/entropy": -129.1576385498047, "objective/non_score_reward": -1.6775317192077637, "objective/rlhf_reward": 10.937702178955078, "objective/scores": 12.625, "policy/approxkl_avg": 0.001147491973824799, "policy/clipfrac_avg": 0.017196711152791977, "loss/policy_avg": -0.6986067295074463, "loss/value_avg": 1.3764233589172363, "val/clipfrac_avg": 0.18510054051876068, "policy/entropy_avg": 0.207261323928833, "val/ratio": 0.9887476563453674, "val/ratio_var": 9.582716302247718e-05, "val/num_eos_tokens": 428, "lr": 7.011695596831102e-06, "episode": 7936, "epoch": 0.40080808080808084, "step": 496}, {"eps": 0, "objective/kl": 32.41982650756836, "objective/entropy": -98.12596130371094, "objective/non_score_reward": -1.6209914684295654, "objective/rlhf_reward": 11.339945793151855, "objective/scores": 12.9375, "policy/approxkl_avg": 0.000857113569509238, "policy/clipfrac_avg": 0.00827649887651205, "loss/policy_avg": -0.18637818098068237, "loss/value_avg": 0.7886095643043518, "val/clipfrac_avg": 0.04437314718961716, "policy/entropy_avg": 0.1563873291015625, "val/ratio": 0.9897803664207458, "val/ratio_var": 3.502453910186887e-05, "val/num_eos_tokens": 728, "lr": 6.999460121598704e-06, "episode": 7952, "epoch": 0.4016161616161616, "step": 497}, {"eps": 0, "objective/kl": 27.433303833007812, "objective/entropy": -148.50958251953125, "objective/non_score_reward": -1.3716652393341064, "objective/rlhf_reward": 11.151771545410156, "objective/scores": 12.5, "policy/approxkl_avg": 0.001251452136784792, "policy/clipfrac_avg": 0.025119343772530556, "loss/policy_avg": 0.5554550290107727, "loss/value_avg": 0.9846082925796509, "val/clipfrac_avg": 0.09154286980628967, "policy/entropy_avg": 0.08732831478118896, "val/ratio": 0.9940745234489441, "val/ratio_var": 3.3608135709073395e-05, "val/num_eos_tokens": 47, "lr": 6.987210377233165e-06, "episode": 7968, "epoch": 0.4024242424242424, "step": 498}, {"eps": 0, "objective/kl": 34.421539306640625, "objective/entropy": -83.75293731689453, "objective/non_score_reward": -1.7210773229599, "objective/rlhf_reward": 12.411735534667969, "objective/scores": 14.125, "policy/approxkl_avg": 0.0008086425950750709, "policy/clipfrac_avg": 0.015222619287669659, "loss/policy_avg": -0.14422039687633514, "loss/value_avg": 0.6888911128044128, "val/clipfrac_avg": 0.2364070564508438, "policy/entropy_avg": 0.1486673355102539, "val/ratio": 0.991155207157135, "val/ratio_var": 5.660337774315849e-05, "val/num_eos_tokens": 388, "lr": 6.974946451154694e-06, "episode": 7984, "epoch": 0.4032323232323232, "step": 499}, {"eps": 0, "objective/kl": 28.21258544921875, "objective/entropy": -83.42674255371094, "objective/non_score_reward": -1.4106292724609375, "objective/rlhf_reward": 12.112808227539062, "objective/scores": 13.5, "policy/approxkl_avg": 0.0009975788416340947, "policy/clipfrac_avg": 0.010807021521031857, "loss/policy_avg": -0.20510384440422058, "loss/value_avg": 0.7008277177810669, "val/clipfrac_avg": 0.08799945563077927, "policy/entropy_avg": 0.15832972526550293, "val/ratio": 0.989881157875061, "val/ratio_var": 5.400492955232039e-05, "val/num_eos_tokens": 98, "lr": 6.962668430884716e-06, "episode": 8000, "epoch": 0.40404040404040403, "step": 500}, {"eps": 0, "objective/kl": 33.63490295410156, "objective/entropy": -98.0992660522461, "objective/non_score_reward": -1.6817452907562256, "objective/rlhf_reward": 11.169817924499512, "objective/scores": 12.875, "policy/approxkl_avg": 0.000961613841354847, "policy/clipfrac_avg": 0.008026218973100185, "loss/policy_avg": 0.11570744216442108, "loss/value_avg": 1.2827966213226318, "val/clipfrac_avg": 0.17985008656978607, "policy/entropy_avg": 0.16532373428344727, "val/ratio": 0.988849937915802, "val/ratio_var": 0.00011337183968862519, "val/num_eos_tokens": 104, "lr": 6.950376404045235e-06, "episode": 8016, "epoch": 0.40484848484848485, "step": 501}, {"eps": 0, "objective/kl": 36.50762176513672, "objective/entropy": -92.8426284790039, "objective/non_score_reward": -1.8253811597824097, "objective/rlhf_reward": 11.557432174682617, "objective/scores": 13.375, "policy/approxkl_avg": 0.0008573720697313547, "policy/clipfrac_avg": 0.008269022218883038, "loss/policy_avg": 0.33835944533348083, "loss/value_avg": 0.479499876499176, "val/clipfrac_avg": 0.021562231704592705, "policy/entropy_avg": 0.14825701713562012, "val/ratio": 0.9906221628189087, "val/ratio_var": 2.253732418466825e-05, "val/num_eos_tokens": 38, "lr": 6.938070458358217e-06, "episode": 8032, "epoch": 0.40565656565656566, "step": 502}, {"eps": 0, "objective/kl": 42.3128776550293, "objective/entropy": -211.23294067382812, "objective/non_score_reward": -2.1156439781188965, "objective/rlhf_reward": 8.333574295043945, "objective/scores": 10.4375, "policy/approxkl_avg": 0.0009327945881523192, "policy/clipfrac_avg": 0.009784913621842861, "loss/policy_avg": -0.003604922443628311, "loss/value_avg": 0.28834861516952515, "val/clipfrac_avg": 0.08479395508766174, "policy/entropy_avg": 0.1659529209136963, "val/ratio": 0.9885441660881042, "val/ratio_var": 8.161830191966146e-05, "val/num_eos_tokens": 600, "lr": 6.925750681644954e-06, "episode": 8048, "epoch": 0.4064646464646465, "step": 503}, {"eps": 0, "objective/kl": 26.93456268310547, "objective/entropy": -71.5333251953125, "objective/non_score_reward": -1.3467280864715576, "objective/rlhf_reward": 11.856396675109863, "objective/scores": 13.1875, "policy/approxkl_avg": 0.0010130183072760701, "policy/clipfrac_avg": 0.016173746436834335, "loss/policy_avg": 0.345672607421875, "loss/value_avg": 0.7462859153747559, "val/clipfrac_avg": 0.06575201451778412, "policy/entropy_avg": 0.18730783462524414, "val/ratio": 0.9880872964859009, "val/ratio_var": 9.104741911869496e-05, "val/num_eos_tokens": 245, "lr": 6.913417161825449e-06, "episode": 8064, "epoch": 0.4072727272727273, "step": 504}, {"eps": 0, "objective/kl": 35.24448013305664, "objective/entropy": -105.62996673583984, "objective/non_score_reward": -1.7622239589691162, "objective/rlhf_reward": 11.698713302612305, "objective/scores": 13.4375, "policy/approxkl_avg": 0.0007742226589471102, "policy/clipfrac_avg": 0.01576562412083149, "loss/policy_avg": 0.5040425062179565, "loss/value_avg": 0.5650883913040161, "val/clipfrac_avg": 0.04690374806523323, "policy/entropy_avg": 0.18837833404541016, "val/ratio": 0.9907522201538086, "val/ratio_var": 3.813532384810969e-05, "val/num_eos_tokens": 570, "lr": 6.90106998691778e-06, "episode": 8080, "epoch": 0.4080808080808081, "step": 505}, {"eps": 0, "objective/kl": 27.575519561767578, "objective/entropy": -108.91736602783203, "objective/non_score_reward": -1.378775954246521, "objective/rlhf_reward": 12.504035949707031, "objective/scores": 13.875, "policy/approxkl_avg": 0.0015673571033403277, "policy/clipfrac_avg": 0.017193179577589035, "loss/policy_avg": -0.13703672587871552, "loss/value_avg": 1.1825928688049316, "val/clipfrac_avg": 0.10861704498529434, "policy/entropy_avg": 0.14080572128295898, "val/ratio": 0.9917985200881958, "val/ratio_var": 0.00015174220607150346, "val/num_eos_tokens": 979, "lr": 6.88870924503747e-06, "episode": 8096, "epoch": 0.4088888888888889, "step": 506}, {"eps": 0, "objective/kl": 27.58624839782715, "objective/entropy": -138.9822998046875, "objective/non_score_reward": -1.3793123960494995, "objective/rlhf_reward": 11.394125938415527, "objective/scores": 12.75, "policy/approxkl_avg": 0.0006402102299034595, "policy/clipfrac_avg": 0.012429703958332539, "loss/policy_avg": -0.4995970129966736, "loss/value_avg": 1.295984148979187, "val/clipfrac_avg": 0.27246493101119995, "policy/entropy_avg": 0.10546708106994629, "val/ratio": 0.9938684701919556, "val/ratio_var": 3.623984594014473e-05, "val/num_eos_tokens": 254, "lr": 6.876335024396872e-06, "episode": 8112, "epoch": 0.40969696969696967, "step": 507}, {"eps": 0, "objective/kl": 24.472135543823242, "objective/entropy": -50.943092346191406, "objective/non_score_reward": -1.223606824874878, "objective/rlhf_reward": 11.54201889038086, "objective/scores": 12.75, "policy/approxkl_avg": 0.0019272379577159882, "policy/clipfrac_avg": 0.021972181275486946, "loss/policy_avg": 0.02236700803041458, "loss/value_avg": 1.0374104976654053, "val/clipfrac_avg": 0.12712614238262177, "policy/entropy_avg": 0.12678146362304688, "val/ratio": 0.991468071937561, "val/ratio_var": 3.564015059964731e-05, "val/num_eos_tokens": 35, "lr": 6.863947413304521e-06, "episode": 8128, "epoch": 0.4105050505050505, "step": 508}, {"eps": 0, "objective/kl": 32.953182220458984, "objective/entropy": -138.50244140625, "objective/non_score_reward": -1.6476593017578125, "objective/rlhf_reward": 11.125778198242188, "objective/scores": 12.75, "policy/approxkl_avg": 0.0009498810395598412, "policy/clipfrac_avg": 0.015503231436014175, "loss/policy_avg": -0.09880562126636505, "loss/value_avg": 0.7543489933013916, "val/clipfrac_avg": 0.08852463960647583, "policy/entropy_avg": 0.13935184478759766, "val/ratio": 0.9911839962005615, "val/ratio_var": 4.137688301852904e-05, "val/num_eos_tokens": 273, "lr": 6.8515465001645184e-06, "episode": 8144, "epoch": 0.4113131313131313, "step": 509}, {"eps": 0, "objective/kl": 32.373985290527344, "objective/entropy": -83.92607116699219, "objective/non_score_reward": -1.618699312210083, "objective/rlhf_reward": 9.771925926208496, "objective/scores": 11.375, "policy/approxkl_avg": 0.000855033693369478, "policy/clipfrac_avg": 0.006832615472376347, "loss/policy_avg": 0.04148940369486809, "loss/value_avg": 0.4635436236858368, "val/clipfrac_avg": 0.1661660373210907, "policy/entropy_avg": 0.14737749099731445, "val/ratio": 0.9900499582290649, "val/ratio_var": 5.550993591896258e-05, "val/num_eos_tokens": 115, "lr": 6.839132373475894e-06, "episode": 8160, "epoch": 0.4121212121212121, "step": 510}, {"eps": 0, "objective/kl": 31.993003845214844, "objective/entropy": -101.56571960449219, "objective/non_score_reward": -1.5996501445770264, "objective/rlhf_reward": 11.556599617004395, "objective/scores": 13.125, "policy/approxkl_avg": 0.0008088398608379066, "policy/clipfrac_avg": 0.014578074216842651, "loss/policy_avg": -0.35513174533843994, "loss/value_avg": 0.916961669921875, "val/clipfrac_avg": 0.14982202649116516, "policy/entropy_avg": 0.16936588287353516, "val/ratio": 0.9902456998825073, "val/ratio_var": 6.044187830411829e-05, "val/num_eos_tokens": 126, "lr": 6.8267051218319766e-06, "episode": 8176, "epoch": 0.4129292929292929, "step": 511}, {"eps": 0, "objective/kl": 28.136348724365234, "objective/entropy": -98.40760803222656, "objective/non_score_reward": -1.4068175554275513, "objective/rlhf_reward": 10.440837860107422, "objective/scores": 11.875, "policy/approxkl_avg": 0.0008492213673889637, "policy/clipfrac_avg": 0.008400412276387215, "loss/policy_avg": 0.004593394696712494, "loss/value_avg": 0.6813367009162903, "val/clipfrac_avg": 0.08613656461238861, "policy/entropy_avg": 0.1722790002822876, "val/ratio": 0.9898751974105835, "val/ratio_var": 4.2340398067608476e-05, "val/num_eos_tokens": 52, "lr": 6.814264833919758e-06, "episode": 8192, "epoch": 0.41373737373737374, "step": 512}, {"eps": 0, "objective/kl": 32.61389923095703, "objective/entropy": -121.57412719726562, "objective/non_score_reward": -1.6306949853897095, "objective/rlhf_reward": 10.650554656982422, "objective/scores": 12.25, "policy/approxkl_avg": 0.0007619216339662671, "policy/clipfrac_avg": 0.008247134275734425, "loss/policy_avg": 0.09692560136318207, "loss/value_avg": 0.45964065194129944, "val/clipfrac_avg": 0.13378626108169556, "policy/entropy_avg": 0.1289653778076172, "val/ratio": 0.9914026260375977, "val/ratio_var": 6.937953730812296e-05, "val/num_eos_tokens": 9, "lr": 6.801811598519268e-06, "episode": 8208, "epoch": 0.41454545454545455, "step": 513}, {"eps": 0, "objective/kl": 32.966880798339844, "objective/entropy": -133.54974365234375, "objective/non_score_reward": -1.6483440399169922, "objective/rlhf_reward": 13.601655960083008, "objective/scores": 15.25, "policy/approxkl_avg": 0.0008372896118089557, "policy/clipfrac_avg": 0.01306083146482706, "loss/policy_avg": -0.16650214791297913, "loss/value_avg": 0.5488893985748291, "val/clipfrac_avg": 0.20459802448749542, "policy/entropy_avg": 0.1169576644897461, "val/ratio": 0.9914659261703491, "val/ratio_var": 3.493462281767279e-05, "val/num_eos_tokens": 21, "lr": 6.789345504502932e-06, "episode": 8224, "epoch": 0.41535353535353536, "step": 514}, {"eps": 0, "objective/kl": 31.566425323486328, "objective/entropy": -105.05879211425781, "objective/non_score_reward": -1.5783213376998901, "objective/rlhf_reward": 10.85136604309082, "objective/scores": 12.4375, "policy/approxkl_avg": 0.0009135222062468529, "policy/clipfrac_avg": 0.011635120026767254, "loss/policy_avg": 0.025405913591384888, "loss/value_avg": 0.4576077163219452, "val/clipfrac_avg": 0.12827220559120178, "policy/entropy_avg": 0.16112518310546875, "val/ratio": 0.9887697696685791, "val/ratio_var": 4.425182851264253e-05, "val/num_eos_tokens": 416, "lr": 6.7768666408349445e-06, "episode": 8240, "epoch": 0.4161616161616162, "step": 515}, {"eps": 0, "objective/kl": 32.33205032348633, "objective/entropy": -104.26789855957031, "objective/non_score_reward": -1.6166024208068848, "objective/rlhf_reward": 10.484960556030273, "objective/scores": 12.125, "policy/approxkl_avg": 0.0010946225374937057, "policy/clipfrac_avg": 0.009625648148357868, "loss/policy_avg": 0.050351403653621674, "loss/value_avg": 0.589043915271759, "val/clipfrac_avg": 0.11179494857788086, "policy/entropy_avg": 0.1612253189086914, "val/ratio": 0.9899939298629761, "val/ratio_var": 6.829272751929238e-05, "val/num_eos_tokens": 386, "lr": 6.764375096570628e-06, "episode": 8256, "epoch": 0.416969696969697, "step": 516}, {"eps": 0, "objective/kl": 33.561153411865234, "objective/entropy": -76.06773376464844, "objective/non_score_reward": -1.6780576705932617, "objective/rlhf_reward": 11.603193283081055, "objective/scores": 13.25, "policy/approxkl_avg": 0.0009749201126396656, "policy/clipfrac_avg": 0.018156636506319046, "loss/policy_avg": 0.016396060585975647, "loss/value_avg": 2.60082745552063, "val/clipfrac_avg": 0.12106801569461823, "policy/entropy_avg": 0.19376277923583984, "val/ratio": 0.9880164861679077, "val/ratio_var": 5.697306551155634e-05, "val/num_eos_tokens": 163, "lr": 6.7518709608558e-06, "episode": 8272, "epoch": 0.4177777777777778, "step": 517}, {"eps": 0, "objective/kl": 27.766971588134766, "objective/entropy": -17.734128952026367, "objective/non_score_reward": -1.3883485794067383, "objective/rlhf_reward": 12.830401420593262, "objective/scores": 14.25, "policy/approxkl_avg": 0.0008711587870493531, "policy/clipfrac_avg": 0.004499074071645737, "loss/policy_avg": -0.07019297033548355, "loss/value_avg": 0.5070562958717346, "val/clipfrac_avg": 0.1293402910232544, "policy/entropy_avg": 0.16478729248046875, "val/ratio": 0.9889296293258667, "val/ratio_var": 3.157386163366027e-05, "val/num_eos_tokens": 145, "lr": 6.739354322926136e-06, "episode": 8288, "epoch": 0.41858585858585856, "step": 518}, {"eps": 0, "objective/kl": 31.432270050048828, "objective/entropy": -62.78154373168945, "objective/non_score_reward": -1.5716133117675781, "objective/rlhf_reward": 12.311199188232422, "objective/scores": 13.875, "policy/approxkl_avg": 0.0008107471512630582, "policy/clipfrac_avg": 0.009700633585453033, "loss/policy_avg": -0.11896735429763794, "loss/value_avg": 0.35987716913223267, "val/clipfrac_avg": 0.12159738689661026, "policy/entropy_avg": 0.12371361255645752, "val/ratio": 0.9906887412071228, "val/ratio_var": 3.533386188792065e-05, "val/num_eos_tokens": 365, "lr": 6.726825272106539e-06, "episode": 8304, "epoch": 0.4193939393939394, "step": 519}, {"eps": 0, "objective/kl": 32.716697692871094, "objective/entropy": -86.20362854003906, "objective/non_score_reward": -1.6358349323272705, "objective/rlhf_reward": 12.606351852416992, "objective/scores": 14.25, "policy/approxkl_avg": 0.0021235994063317776, "policy/clipfrac_avg": 0.015771860256791115, "loss/policy_avg": -0.0776340663433075, "loss/value_avg": 0.3134997487068176, "val/clipfrac_avg": 0.05747883766889572, "policy/entropy_avg": 0.1624317169189453, "val/ratio": 0.989377498626709, "val/ratio_var": 5.577542833634652e-05, "val/num_eos_tokens": 30, "lr": 6.714283897810488e-06, "episode": 8320, "epoch": 0.4202020202020202, "step": 520}, {"eps": 0, "objective/kl": 23.357372283935547, "objective/entropy": -131.93936157226562, "objective/non_score_reward": -1.1678686141967773, "objective/rlhf_reward": 12.738380432128906, "objective/scores": 13.875, "policy/approxkl_avg": 0.0008331306744366884, "policy/clipfrac_avg": 0.018915321677923203, "loss/policy_avg": -0.464477002620697, "loss/value_avg": 1.1038016080856323, "val/clipfrac_avg": 0.1614842414855957, "policy/entropy_avg": 0.14388728141784668, "val/ratio": 0.9908460378646851, "val/ratio_var": 6.456010305555537e-05, "val/num_eos_tokens": 377, "lr": 6.701730289539417e-06, "episode": 8336, "epoch": 0.421010101010101, "step": 521}, {"eps": 0, "objective/kl": 27.40955352783203, "objective/entropy": -114.08953094482422, "objective/non_score_reward": -1.3704776763916016, "objective/rlhf_reward": 11.652959823608398, "objective/scores": 13.0, "policy/approxkl_avg": 0.000854157842695713, "policy/clipfrac_avg": 0.015246449038386345, "loss/policy_avg": -0.130820631980896, "loss/value_avg": 0.6008985042572021, "val/clipfrac_avg": 0.08382974565029144, "policy/entropy_avg": 0.1589183807373047, "val/ratio": 0.9912443161010742, "val/ratio_var": 5.435436833067797e-05, "val/num_eos_tokens": 337, "lr": 6.689164536882059e-06, "episode": 8352, "epoch": 0.4218181818181818, "step": 522}, {"eps": 0, "objective/kl": 34.11695098876953, "objective/entropy": -129.62985229492188, "objective/non_score_reward": -1.7058476209640503, "objective/rlhf_reward": 10.251184463500977, "objective/scores": 11.9375, "policy/approxkl_avg": 0.0007716008694842458, "policy/clipfrac_avg": 0.0059022363275289536, "loss/policy_avg": -0.1433180719614029, "loss/value_avg": 0.31795254349708557, "val/clipfrac_avg": 0.14383307099342346, "policy/entropy_avg": 0.14097148180007935, "val/ratio": 0.9905247092247009, "val/ratio_var": 4.685216845246032e-05, "val/num_eos_tokens": 34, "lr": 6.676586729513823e-06, "episode": 8368, "epoch": 0.4226262626262626, "step": 523}, {"eps": 0, "objective/kl": 30.304183959960938, "objective/entropy": -46.01490783691406, "objective/non_score_reward": -1.5152091979980469, "objective/rlhf_reward": 13.375415802001953, "objective/scores": 14.875, "policy/approxkl_avg": 0.0009743375121615827, "policy/clipfrac_avg": 0.008508563973009586, "loss/policy_avg": 0.09673983603715897, "loss/value_avg": 0.37219128012657166, "val/clipfrac_avg": 0.07922305911779404, "policy/entropy_avg": 0.19330501556396484, "val/ratio": 0.9889954328536987, "val/ratio_var": 0.00010105717228725553, "val/num_eos_tokens": 4, "lr": 6.6639969571961406e-06, "episode": 8384, "epoch": 0.42343434343434344, "step": 524}, {"eps": 0, "objective/kl": 26.15176010131836, "objective/entropy": -73.01426696777344, "objective/non_score_reward": -1.30758798122406, "objective/rlhf_reward": 12.481473922729492, "objective/scores": 13.8125, "policy/approxkl_avg": 0.0008940043626353145, "policy/clipfrac_avg": 0.011357388459146023, "loss/policy_avg": 0.42921364307403564, "loss/value_avg": 0.41969558596611023, "val/clipfrac_avg": 0.04973672330379486, "policy/entropy_avg": 0.18736028671264648, "val/ratio": 0.9906824827194214, "val/ratio_var": 4.821328184334561e-05, "val/num_eos_tokens": 32, "lr": 6.651395309775837e-06, "episode": 8400, "epoch": 0.42424242424242425, "step": 525}, {"eps": 0, "objective/kl": 29.63160514831543, "objective/entropy": -122.70181274414062, "objective/non_score_reward": -1.4815802574157715, "objective/rlhf_reward": 10.88560676574707, "objective/scores": 12.375, "policy/approxkl_avg": 0.0008636523270979524, "policy/clipfrac_avg": 0.01251271739602089, "loss/policy_avg": 0.1258348971605301, "loss/value_avg": 0.46568286418914795, "val/clipfrac_avg": 0.03258850425481796, "policy/entropy_avg": 0.16792958974838257, "val/ratio": 0.9903141260147095, "val/ratio_var": 3.827890759566799e-05, "val/num_eos_tokens": 132, "lr": 6.638781877184477e-06, "episode": 8416, "epoch": 0.42505050505050507, "step": 526}, {"eps": 0, "objective/kl": 26.3393611907959, "objective/entropy": -157.23638916015625, "objective/non_score_reward": -1.3169679641723633, "objective/rlhf_reward": 12.40959358215332, "objective/scores": 13.75, "policy/approxkl_avg": 0.0009735558414831758, "policy/clipfrac_avg": 0.007334539666771889, "loss/policy_avg": 0.0722554475069046, "loss/value_avg": 1.4936760663986206, "val/clipfrac_avg": 0.08778189867734909, "policy/entropy_avg": 0.12781143188476562, "val/ratio": 0.993151068687439, "val/ratio_var": 0.00014699323219247162, "val/num_eos_tokens": 42, "lr": 6.626156749437736e-06, "episode": 8432, "epoch": 0.4258585858585859, "step": 527}, {"eps": 0, "objective/kl": 39.47442626953125, "objective/entropy": -100.36758422851562, "objective/non_score_reward": -1.9737212657928467, "objective/rlhf_reward": 10.36221694946289, "objective/scores": 12.3125, "policy/approxkl_avg": 0.0009299812954850495, "policy/clipfrac_avg": 0.011641680262982845, "loss/policy_avg": 0.2405271977186203, "loss/value_avg": 0.30028170347213745, "val/clipfrac_avg": 0.17108477652072906, "policy/entropy_avg": 0.16964960098266602, "val/ratio": 0.9882971048355103, "val/ratio_var": 5.371840961743146e-05, "val/num_eos_tokens": 247, "lr": 6.6135200166347505e-06, "episode": 8448, "epoch": 0.4266666666666667, "step": 528}, {"eps": 0, "objective/kl": 30.68231964111328, "objective/entropy": -314.8489990234375, "objective/non_score_reward": -1.5341161489486694, "objective/rlhf_reward": 11.434633255004883, "objective/scores": 13.0, "policy/approxkl_avg": 0.0009698778740130365, "policy/clipfrac_avg": 0.01093447208404541, "loss/policy_avg": -0.15250246226787567, "loss/value_avg": 0.8714890480041504, "val/clipfrac_avg": 0.09076748788356781, "policy/entropy_avg": 0.10747766494750977, "val/ratio": 0.9922375679016113, "val/ratio_var": 6.258123903535306e-05, "val/num_eos_tokens": 650, "lr": 6.600871768957474e-06, "episode": 8464, "epoch": 0.42747474747474745, "step": 529}, {"eps": 0, "objective/kl": 41.18302917480469, "objective/entropy": -109.72494506835938, "objective/non_score_reward": -2.0591516494750977, "objective/rlhf_reward": 11.933036804199219, "objective/scores": 14.0, "policy/approxkl_avg": 0.0007708189077675343, "policy/clipfrac_avg": 0.005906249396502972, "loss/policy_avg": -0.35478073358535767, "loss/value_avg": 0.8044853210449219, "val/clipfrac_avg": 0.06992866098880768, "policy/entropy_avg": 0.14908885955810547, "val/ratio": 0.9903333187103271, "val/ratio_var": 7.135802297852933e-05, "val/num_eos_tokens": 571, "lr": 6.588212096670037e-06, "episode": 8480, "epoch": 0.42828282828282827, "step": 530}, {"eps": 0, "objective/kl": 32.64395523071289, "objective/entropy": -76.89524841308594, "objective/non_score_reward": -1.6321979761123657, "objective/rlhf_reward": 10.934207916259766, "objective/scores": 12.5625, "policy/approxkl_avg": 0.0009906484046950936, "policy/clipfrac_avg": 0.006618197076022625, "loss/policy_avg": 0.08454768359661102, "loss/value_avg": 0.4138789176940918, "val/clipfrac_avg": 0.08779165148735046, "policy/entropy_avg": 0.1799774169921875, "val/ratio": 0.9877060651779175, "val/ratio_var": 5.0242877478012815e-05, "val/num_eos_tokens": 947, "lr": 6.575541090118105e-06, "episode": 8496, "epoch": 0.4290909090909091, "step": 531}, {"eps": 0, "objective/kl": 31.940135955810547, "objective/entropy": -62.30128860473633, "objective/non_score_reward": -1.597006916999817, "objective/rlhf_reward": 12.582681655883789, "objective/scores": 14.1875, "policy/approxkl_avg": 0.0010326767805963755, "policy/clipfrac_avg": 0.005477210506796837, "loss/policy_avg": -0.6856231093406677, "loss/value_avg": 0.972754180431366, "val/clipfrac_avg": 0.1998615562915802, "policy/entropy_avg": 0.1778457760810852, "val/ratio": 0.9881020188331604, "val/ratio_var": 5.7397228374611586e-05, "val/num_eos_tokens": 38, "lr": 6.562858839728224e-06, "episode": 8512, "epoch": 0.4298989898989899, "step": 532}, {"eps": 0, "objective/kl": 24.379880905151367, "objective/entropy": -143.62107849121094, "objective/non_score_reward": -1.218994140625, "objective/rlhf_reward": 12.210693359375, "objective/scores": 13.4375, "policy/approxkl_avg": 0.0008440241799689829, "policy/clipfrac_avg": 0.009614679962396622, "loss/policy_avg": -0.00426415354013443, "loss/value_avg": 0.3097630739212036, "val/clipfrac_avg": 0.052517104893922806, "policy/entropy_avg": 0.13739681243896484, "val/ratio": 0.9900696277618408, "val/ratio_var": 6.144143844721839e-05, "val/num_eos_tokens": 10, "lr": 6.550165436007185e-06, "episode": 8528, "epoch": 0.4307070707070707, "step": 533}, {"eps": 0, "objective/kl": 30.961679458618164, "objective/entropy": -39.871124267578125, "objective/non_score_reward": -1.548084020614624, "objective/rlhf_reward": 13.412853240966797, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0011838905047625303, "policy/clipfrac_avg": 0.01087412890046835, "loss/policy_avg": -0.09833642840385437, "loss/value_avg": 0.3559184670448303, "val/clipfrac_avg": 0.11940991878509521, "policy/entropy_avg": 0.18686485290527344, "val/ratio": 0.9873068928718567, "val/ratio_var": 5.543338193092495e-05, "val/num_eos_tokens": 60, "lr": 6.537460969541378e-06, "episode": 8544, "epoch": 0.4315151515151515, "step": 534}, {"eps": 0, "objective/kl": 39.486045837402344, "objective/entropy": -58.27792739868164, "objective/non_score_reward": -1.9743022918701172, "objective/rlhf_reward": 10.916322708129883, "objective/scores": 12.875, "policy/approxkl_avg": 0.0008673222037032247, "policy/clipfrac_avg": 0.0051139891147613525, "loss/policy_avg": 0.18861787021160126, "loss/value_avg": 0.3709791898727417, "val/clipfrac_avg": 0.19597014784812927, "policy/entropy_avg": 0.1625194549560547, "val/ratio": 0.9891222715377808, "val/ratio_var": 2.865192072931677e-05, "val/num_eos_tokens": 189, "lr": 6.524745530996137e-06, "episode": 8560, "epoch": 0.43232323232323233, "step": 535}, {"eps": 0, "objective/kl": 28.339763641357422, "objective/entropy": -41.29559326171875, "objective/non_score_reward": -1.4169882535934448, "objective/rlhf_reward": 12.317386627197266, "objective/scores": 13.75, "policy/approxkl_avg": 0.0007994387997314334, "policy/clipfrac_avg": 0.01735379919409752, "loss/policy_avg": -0.030973181128501892, "loss/value_avg": 0.6104284524917603, "val/clipfrac_avg": 0.061991170048713684, "policy/entropy_avg": 0.17484283447265625, "val/ratio": 0.9900209903717041, "val/ratio_var": 5.226740540820174e-05, "val/num_eos_tokens": 158, "lr": 6.512019211115099e-06, "episode": 8576, "epoch": 0.43313131313131314, "step": 536}, {"eps": 0, "objective/kl": 36.661048889160156, "objective/entropy": -98.1541976928711, "objective/non_score_reward": -1.833052396774292, "objective/rlhf_reward": 11.971634864807129, "objective/scores": 13.8125, "policy/approxkl_avg": 0.0011100426781922579, "policy/clipfrac_avg": 0.009958980605006218, "loss/policy_avg": -0.14375048875808716, "loss/value_avg": 0.18352828919887543, "val/clipfrac_avg": 0.05129721760749817, "policy/entropy_avg": 0.14308547973632812, "val/ratio": 0.9901905059814453, "val/ratio_var": 3.793285577557981e-05, "val/num_eos_tokens": 479, "lr": 6.499282100719558e-06, "episode": 8592, "epoch": 0.43393939393939396, "step": 537}, {"eps": 0, "objective/kl": 31.673198699951172, "objective/entropy": -113.62753295898438, "objective/non_score_reward": -1.5836598873138428, "objective/rlhf_reward": 12.431964874267578, "objective/scores": 14.0, "policy/approxkl_avg": 0.0014390661381185055, "policy/clipfrac_avg": 0.02696000598371029, "loss/policy_avg": -0.06621083617210388, "loss/value_avg": 0.621555745601654, "val/clipfrac_avg": 0.10710769891738892, "policy/entropy_avg": 0.13946056365966797, "val/ratio": 0.9896073341369629, "val/ratio_var": 9.375382796861231e-05, "val/num_eos_tokens": 342, "lr": 6.486534290707815e-06, "episode": 8608, "epoch": 0.43474747474747477, "step": 538}, {"eps": 0, "objective/kl": 34.75250244140625, "objective/entropy": -120.77999877929688, "objective/non_score_reward": -1.7376251220703125, "objective/rlhf_reward": 11.762374877929688, "objective/scores": 13.5, "policy/approxkl_avg": 0.0009148735553026199, "policy/clipfrac_avg": 0.005174038931727409, "loss/policy_avg": -0.06200864911079407, "loss/value_avg": 0.3554137051105499, "val/clipfrac_avg": 0.06834189593791962, "policy/entropy_avg": 0.14241600036621094, "val/ratio": 0.9897119998931885, "val/ratio_var": 5.696159860235639e-05, "val/num_eos_tokens": 484, "lr": 6.473775872054522e-06, "episode": 8624, "epoch": 0.43555555555555553, "step": 539}, {"eps": 0, "objective/kl": 40.74346160888672, "objective/entropy": -115.64312744140625, "objective/non_score_reward": -2.03717303276062, "objective/rlhf_reward": 10.947202682495117, "objective/scores": 13.0, "policy/approxkl_avg": 0.0008006396819837391, "policy/clipfrac_avg": 0.006549537181854248, "loss/policy_avg": 0.060983721166849136, "loss/value_avg": 0.4710575342178345, "val/clipfrac_avg": 0.15336185693740845, "policy/entropy_avg": 0.141357421875, "val/ratio": 0.9915748238563538, "val/ratio_var": 5.314065856509842e-05, "val/num_eos_tokens": 624, "lr": 6.461006935810048e-06, "episode": 8640, "epoch": 0.43636363636363634, "step": 540}, {"eps": 0, "objective/kl": 31.47679901123047, "objective/entropy": -112.73287963867188, "objective/non_score_reward": -1.5738401412963867, "objective/rlhf_reward": 13.254284858703613, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0009382686112076044, "policy/clipfrac_avg": 0.0064744651317596436, "loss/policy_avg": -0.39746806025505066, "loss/value_avg": 1.7929481267929077, "val/clipfrac_avg": 0.2390729784965515, "policy/entropy_avg": 0.15688323974609375, "val/ratio": 0.9905893802642822, "val/ratio_var": 9.629622945794836e-05, "val/num_eos_tokens": 1255, "lr": 6.448227573099815e-06, "episode": 8656, "epoch": 0.43717171717171716, "step": 541}, {"eps": 0, "objective/kl": 33.559234619140625, "objective/entropy": -104.77660369873047, "objective/non_score_reward": -1.677961826324463, "objective/rlhf_reward": 13.173601150512695, "objective/scores": 14.875, "policy/approxkl_avg": 0.0010652798227965832, "policy/clipfrac_avg": 0.005728135816752911, "loss/policy_avg": -0.2003842294216156, "loss/value_avg": 0.4203144907951355, "val/clipfrac_avg": 0.1355896294116974, "policy/entropy_avg": 0.1696929931640625, "val/ratio": 0.9887447953224182, "val/ratio_var": 5.705601142835803e-05, "val/num_eos_tokens": 1683, "lr": 6.435437875123658e-06, "episode": 8672, "epoch": 0.43797979797979797, "step": 542}, {"eps": 0, "objective/kl": 29.884620666503906, "objective/entropy": -78.30276489257812, "objective/non_score_reward": -1.4942309856414795, "objective/rlhf_reward": 13.052643775939941, "objective/scores": 14.5625, "policy/approxkl_avg": 0.0008832053281366825, "policy/clipfrac_avg": 0.010442769154906273, "loss/policy_avg": -0.05463821440935135, "loss/value_avg": 0.33894190192222595, "val/clipfrac_avg": 0.05107077211141586, "policy/entropy_avg": 0.1459188461303711, "val/ratio": 0.9910210967063904, "val/ratio_var": 3.290848690085113e-05, "val/num_eos_tokens": 112, "lr": 6.4226379331551625e-06, "episode": 8688, "epoch": 0.4387878787878788, "step": 543}, {"eps": 0, "objective/kl": 41.26933288574219, "objective/entropy": -48.91859817504883, "objective/non_score_reward": -2.063466787338257, "objective/rlhf_reward": 12.561532974243164, "objective/scores": 14.625, "policy/approxkl_avg": 0.0008429911686107516, "policy/clipfrac_avg": 0.006084254011511803, "loss/policy_avg": 0.16157524287700653, "loss/value_avg": 0.43978190422058105, "val/clipfrac_avg": 0.0520780012011528, "policy/entropy_avg": 0.15523910522460938, "val/ratio": 0.990166187286377, "val/ratio_var": 4.473194348975085e-05, "val/num_eos_tokens": 150, "lr": 6.409827838541029e-06, "episode": 8704, "epoch": 0.4395959595959596, "step": 544}, {"eps": 0, "objective/kl": 35.155426025390625, "objective/entropy": -105.18928527832031, "objective/non_score_reward": -1.7577712535858154, "objective/rlhf_reward": 14.875041961669922, "objective/scores": 16.625, "policy/approxkl_avg": 0.0011077180970460176, "policy/clipfrac_avg": 0.010063378140330315, "loss/policy_avg": -0.15311197936534882, "loss/value_avg": 0.4987826645374298, "val/clipfrac_avg": 0.10631218552589417, "policy/entropy_avg": 0.17020034790039062, "val/ratio": 0.9899839758872986, "val/ratio_var": 7.999869558261707e-05, "val/num_eos_tokens": 501, "lr": 6.397007682700407e-06, "episode": 8720, "epoch": 0.4404040404040404, "step": 545}, {"eps": 0, "objective/kl": 31.340700149536133, "objective/entropy": -31.199787139892578, "objective/non_score_reward": -1.5670350790023804, "objective/rlhf_reward": 11.190776824951172, "objective/scores": 12.75, "policy/approxkl_avg": 0.0009682231466285884, "policy/clipfrac_avg": 0.006479987874627113, "loss/policy_avg": 0.1040591150522232, "loss/value_avg": 0.4214034080505371, "val/clipfrac_avg": 0.1099347323179245, "policy/entropy_avg": 0.20100021362304688, "val/ratio": 0.9890173673629761, "val/ratio_var": 3.694462066050619e-05, "val/num_eos_tokens": 238, "lr": 6.384177557124247e-06, "episode": 8736, "epoch": 0.4412121212121212, "step": 546}, {"eps": 0, "objective/kl": 34.65884780883789, "objective/entropy": -39.132110595703125, "objective/non_score_reward": -1.7329423427581787, "objective/rlhf_reward": 9.946744918823242, "objective/scores": 11.6875, "policy/approxkl_avg": 0.000952253001742065, "policy/clipfrac_avg": 0.01447022333741188, "loss/policy_avg": 0.5359160304069519, "loss/value_avg": 1.8251168727874756, "val/clipfrac_avg": 0.09755296260118484, "policy/entropy_avg": 0.18070626258850098, "val/ratio": 0.9890221953392029, "val/ratio_var": 4.6701326937181875e-05, "val/num_eos_tokens": 147, "lr": 6.3713375533746525e-06, "episode": 8752, "epoch": 0.44202020202020204, "step": 547}, {"eps": 0, "objective/kl": 37.905120849609375, "objective/entropy": -175.40753173828125, "objective/non_score_reward": -1.8952560424804688, "objective/rlhf_reward": 12.608650207519531, "objective/scores": 14.5, "policy/approxkl_avg": 0.000838284962810576, "policy/clipfrac_avg": 0.003946049604564905, "loss/policy_avg": 0.003339662216603756, "loss/value_avg": 0.3004421889781952, "val/clipfrac_avg": 0.07305619865655899, "policy/entropy_avg": 0.15326714515686035, "val/ratio": 0.9898229837417603, "val/ratio_var": 4.8889804020291194e-05, "val/num_eos_tokens": 253, "lr": 6.358487763084219e-06, "episode": 8768, "epoch": 0.44282828282828285, "step": 548}, {"eps": 0, "objective/kl": 31.03716278076172, "objective/entropy": -127.72113037109375, "objective/non_score_reward": -1.5518580675125122, "objective/rlhf_reward": 13.635642051696777, "objective/scores": 15.1875, "policy/approxkl_avg": 0.000858222832903266, "policy/clipfrac_avg": 0.012858349829912186, "loss/policy_avg": 0.07848238945007324, "loss/value_avg": 1.0936524868011475, "val/clipfrac_avg": 0.10532164573669434, "policy/entropy_avg": 0.1750965118408203, "val/ratio": 0.990279495716095, "val/ratio_var": 5.98018268647138e-05, "val/num_eos_tokens": 707, "lr": 6.345628277955384e-06, "episode": 8784, "epoch": 0.44363636363636366, "step": 549}, {"eps": 0, "objective/kl": 30.36469268798828, "objective/entropy": -142.31295776367188, "objective/non_score_reward": -1.5182347297668457, "objective/rlhf_reward": 13.255203247070312, "objective/scores": 14.75, "policy/approxkl_avg": 0.0009378796094097197, "policy/clipfrac_avg": 0.00806361623108387, "loss/policy_avg": -0.13267037272453308, "loss/value_avg": 0.6979200839996338, "val/clipfrac_avg": 0.21628013253211975, "policy/entropy_avg": 0.1446208953857422, "val/ratio": 0.9899749755859375, "val/ratio_var": 7.445859955623746e-05, "val/num_eos_tokens": 1703, "lr": 6.3327591897597765e-06, "episode": 8800, "epoch": 0.4444444444444444, "step": 550}, {"eps": 0, "objective/kl": 28.549150466918945, "objective/entropy": -74.91482543945312, "objective/non_score_reward": -1.4274574518203735, "objective/rlhf_reward": 13.556917190551758, "objective/scores": 15.0, "policy/approxkl_avg": 0.0011112524662166834, "policy/clipfrac_avg": 0.011593078263103962, "loss/policy_avg": 0.15575791895389557, "loss/value_avg": 0.4495770335197449, "val/clipfrac_avg": 0.09938865900039673, "policy/entropy_avg": 0.17023122310638428, "val/ratio": 0.9868941903114319, "val/ratio_var": 0.00012118481390643865, "val/num_eos_tokens": 124, "lr": 6.319880590337549e-06, "episode": 8816, "epoch": 0.44525252525252523, "step": 551}, {"eps": 0, "objective/kl": 38.20920944213867, "objective/entropy": -101.52589416503906, "objective/non_score_reward": -1.9104604721069336, "objective/rlhf_reward": 13.23016357421875, "objective/scores": 15.125, "policy/approxkl_avg": 0.0010805606143549085, "policy/clipfrac_avg": 0.012465233914554119, "loss/policy_avg": -0.158098965883255, "loss/value_avg": 0.5848870277404785, "val/clipfrac_avg": 0.03225297853350639, "policy/entropy_avg": 0.1814594268798828, "val/ratio": 0.9882332682609558, "val/ratio_var": 5.3187635785434395e-05, "val/num_eos_tokens": 138, "lr": 6.306992571596742e-06, "episode": 8832, "epoch": 0.44606060606060605, "step": 552}, {"eps": 0, "objective/kl": 36.24083709716797, "objective/entropy": -113.32449340820312, "objective/non_score_reward": -1.8120418787002563, "objective/rlhf_reward": 11.742645263671875, "objective/scores": 13.5625, "policy/approxkl_avg": 0.0010494563030079007, "policy/clipfrac_avg": 0.012232597917318344, "loss/policy_avg": 0.11628187447786331, "loss/value_avg": 0.6404980421066284, "val/clipfrac_avg": 0.14791586995124817, "policy/entropy_avg": 0.16427016258239746, "val/ratio": 0.9883185625076294, "val/ratio_var": 6.705433770548552e-05, "val/num_eos_tokens": 22, "lr": 6.294095225512604e-06, "episode": 8848, "epoch": 0.44686868686868686, "step": 553}, {"eps": 0, "objective/kl": 36.22610092163086, "objective/entropy": -56.46511459350586, "objective/non_score_reward": -1.811305046081543, "objective/rlhf_reward": 10.868382453918457, "objective/scores": 12.6875, "policy/approxkl_avg": 0.0009609910775907338, "policy/clipfrac_avg": 0.020421186462044716, "loss/policy_avg": 0.5528974533081055, "loss/value_avg": 0.9337465167045593, "val/clipfrac_avg": 0.02570945769548416, "policy/entropy_avg": 0.21524810791015625, "val/ratio": 0.9899651408195496, "val/ratio_var": 2.9964650821057148e-05, "val/num_eos_tokens": 5, "lr": 6.281188644126961e-06, "episode": 8864, "epoch": 0.4476767676767677, "step": 554}, {"eps": 0, "objective/kl": 43.30995559692383, "objective/entropy": -92.14622497558594, "objective/non_score_reward": -2.1654977798461914, "objective/rlhf_reward": 12.537628173828125, "objective/scores": 14.6875, "policy/approxkl_avg": 0.001519187237136066, "policy/clipfrac_avg": 0.008950314484536648, "loss/policy_avg": -0.13656526803970337, "loss/value_avg": 0.38830268383026123, "val/clipfrac_avg": 0.12284453213214874, "policy/entropy_avg": 0.16514253616333008, "val/ratio": 0.9889045357704163, "val/ratio_var": 9.022261656355113e-05, "val/num_eos_tokens": 564, "lr": 6.268272919547537e-06, "episode": 8880, "epoch": 0.4484848484848485, "step": 555}, {"eps": 0, "objective/kl": 42.43487548828125, "objective/entropy": -117.01277923583984, "objective/non_score_reward": -2.12174391746521, "objective/rlhf_reward": 12.104818344116211, "objective/scores": 14.25, "policy/approxkl_avg": 0.0008287889650091529, "policy/clipfrac_avg": 0.008630437776446342, "loss/policy_avg": -0.2719430923461914, "loss/value_avg": 0.5197917222976685, "val/clipfrac_avg": 0.06423404067754745, "policy/entropy_avg": 0.13199996948242188, "val/ratio": 0.9903758764266968, "val/ratio_var": 5.3244064474711195e-05, "val/num_eos_tokens": 354, "lr": 6.255348143947313e-06, "episode": 8896, "epoch": 0.4492929292929293, "step": 556}, {"eps": 0, "objective/kl": 42.79301452636719, "objective/entropy": -73.03179931640625, "objective/non_score_reward": -2.139650821685791, "objective/rlhf_reward": 12.75878620147705, "objective/scores": 14.875, "policy/approxkl_avg": 0.0008902574772946537, "policy/clipfrac_avg": 0.006880009081214666, "loss/policy_avg": -0.0356672927737236, "loss/value_avg": 0.18395909667015076, "val/clipfrac_avg": 0.0390281043946743, "policy/entropy_avg": 0.16189002990722656, "val/ratio": 0.9893250465393066, "val/ratio_var": 4.1720588342286646e-05, "val/num_eos_tokens": 239, "lr": 6.242414409563855e-06, "episode": 8912, "epoch": 0.4501010101010101, "step": 557}, {"eps": 0, "objective/kl": 44.569541931152344, "objective/entropy": -104.41824340820312, "objective/non_score_reward": -2.2284772396087646, "objective/rlhf_reward": 13.107460021972656, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0009092853288166225, "policy/clipfrac_avg": 0.010719258338212967, "loss/policy_avg": 0.02715390920639038, "loss/value_avg": 0.41223347187042236, "val/clipfrac_avg": 0.03966514393687248, "policy/entropy_avg": 0.13871383666992188, "val/ratio": 0.9900314807891846, "val/ratio_var": 5.0811173423426226e-05, "val/num_eos_tokens": 642, "lr": 6.229471808698673e-06, "episode": 8928, "epoch": 0.4509090909090909, "step": 558}, {"eps": 0, "objective/kl": 46.451202392578125, "objective/entropy": -57.53273010253906, "objective/non_score_reward": -2.3225603103637695, "objective/rlhf_reward": 11.65400218963623, "objective/scores": 14.0, "policy/approxkl_avg": 0.0009985085343942046, "policy/clipfrac_avg": 0.009565187618136406, "loss/policy_avg": 0.1378185749053955, "loss/value_avg": 0.5425667762756348, "val/clipfrac_avg": 0.14630571007728577, "policy/entropy_avg": 0.16213226318359375, "val/ratio": 0.9897133111953735, "val/ratio_var": 4.261602953192778e-05, "val/num_eos_tokens": 76, "lr": 6.216520433716544e-06, "episode": 8944, "epoch": 0.45171717171717174, "step": 559}, {"eps": 0, "objective/kl": 35.97865676879883, "objective/entropy": -166.40460205078125, "objective/non_score_reward": -1.7989329099655151, "objective/rlhf_reward": 11.099504470825195, "objective/scores": 12.875, "policy/approxkl_avg": 0.0014841826632618904, "policy/clipfrac_avg": 0.011655549518764019, "loss/policy_avg": -0.009004142135381699, "loss/value_avg": 0.3125573992729187, "val/clipfrac_avg": 0.06238756328821182, "policy/entropy_avg": 0.15684080123901367, "val/ratio": 0.9896181225776672, "val/ratio_var": 8.536590758012608e-05, "val/num_eos_tokens": 381, "lr": 6.2035603770448664e-06, "episode": 8960, "epoch": 0.45252525252525255, "step": 560}, {"eps": 0, "objective/kl": 30.727413177490234, "objective/entropy": -117.21914672851562, "objective/non_score_reward": -1.5363706350326538, "objective/rlhf_reward": 11.635503768920898, "objective/scores": 13.1875, "policy/approxkl_avg": 0.001313843298703432, "policy/clipfrac_avg": 0.013457929715514183, "loss/policy_avg": -0.011498738080263138, "loss/value_avg": 0.5593593716621399, "val/clipfrac_avg": 0.06608335673809052, "policy/entropy_avg": 0.16321659088134766, "val/ratio": 0.9888886213302612, "val/ratio_var": 6.117957673268393e-05, "val/num_eos_tokens": 5, "lr": 6.1905917311729915e-06, "episode": 8976, "epoch": 0.4533333333333333, "step": 561}, {"eps": 0, "objective/kl": 36.154850006103516, "objective/entropy": -31.294681549072266, "objective/non_score_reward": -1.8077425956726074, "objective/rlhf_reward": 12.184444427490234, "objective/scores": 14.0, "policy/approxkl_avg": 0.0010273936204612255, "policy/clipfrac_avg": 0.007020684890449047, "loss/policy_avg": -0.0510425865650177, "loss/value_avg": 0.3040882349014282, "val/clipfrac_avg": 0.06005093827843666, "policy/entropy_avg": 0.1751842498779297, "val/ratio": 0.9871533513069153, "val/ratio_var": 6.729259621351957e-05, "val/num_eos_tokens": 162, "lr": 6.177614588651572e-06, "episode": 8992, "epoch": 0.4541414141414141, "step": 562}, {"eps": 0, "objective/kl": 36.547935485839844, "objective/entropy": -124.56060791015625, "objective/non_score_reward": -1.8273968696594238, "objective/rlhf_reward": 13.203852653503418, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008937460370361805, "policy/clipfrac_avg": 0.01175684854388237, "loss/policy_avg": 0.03235238790512085, "loss/value_avg": 0.7062409520149231, "val/clipfrac_avg": 0.08029687404632568, "policy/entropy_avg": 0.13650989532470703, "val/ratio": 0.991959810256958, "val/ratio_var": 6.220470095286146e-05, "val/num_eos_tokens": 503, "lr": 6.164629042091894e-06, "episode": 9008, "epoch": 0.45494949494949494, "step": 563}, {"eps": 0, "objective/kl": 33.58654022216797, "objective/entropy": -140.72860717773438, "objective/non_score_reward": -1.679327130317688, "objective/rlhf_reward": 11.437860488891602, "objective/scores": 13.125, "policy/approxkl_avg": 0.0007259284611791372, "policy/clipfrac_avg": 0.008242369629442692, "loss/policy_avg": 0.0829654112458229, "loss/value_avg": 0.461310476064682, "val/clipfrac_avg": 0.04573877528309822, "policy/entropy_avg": 0.1111001968383789, "val/ratio": 0.9929757118225098, "val/ratio_var": 2.9616816391353495e-05, "val/num_eos_tokens": 393, "lr": 6.151635184165219e-06, "episode": 9024, "epoch": 0.45575757575757575, "step": 564}, {"eps": 0, "objective/kl": 38.57415771484375, "objective/entropy": -49.62328338623047, "objective/non_score_reward": -1.9287079572677612, "objective/rlhf_reward": 14.235355377197266, "objective/scores": 16.125, "policy/approxkl_avg": 0.0010976013727486134, "policy/clipfrac_avg": 0.004873812198638916, "loss/policy_avg": -0.13684070110321045, "loss/value_avg": 0.2872938811779022, "val/clipfrac_avg": 0.21986615657806396, "policy/entropy_avg": 0.19704437255859375, "val/ratio": 0.9863287210464478, "val/ratio_var": 5.253278141026385e-05, "val/num_eos_tokens": 90, "lr": 6.1386331076021234e-06, "episode": 9040, "epoch": 0.45656565656565656, "step": 565}, {"eps": 0, "objective/kl": 40.845550537109375, "objective/entropy": -66.99671936035156, "objective/non_score_reward": -2.0422775745391846, "objective/rlhf_reward": 12.223346710205078, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009532993426546454, "policy/clipfrac_avg": 0.008950391784310341, "loss/policy_avg": 0.15097515285015106, "loss/value_avg": 0.8962790966033936, "val/clipfrac_avg": 0.16976982355117798, "policy/entropy_avg": 0.15424060821533203, "val/ratio": 0.9896418452262878, "val/ratio_var": 6.317990482784808e-05, "val/num_eos_tokens": 139, "lr": 6.125622905191835e-06, "episode": 9056, "epoch": 0.4573737373737374, "step": 566}, {"eps": 0, "objective/kl": 31.181835174560547, "objective/entropy": -69.26618957519531, "objective/non_score_reward": -1.5590916872024536, "objective/rlhf_reward": 11.479970932006836, "objective/scores": 13.0625, "policy/approxkl_avg": 0.0010285003809258342, "policy/clipfrac_avg": 0.007737594656646252, "loss/policy_avg": 0.0012879595160484314, "loss/value_avg": 0.6243962049484253, "val/clipfrac_avg": 0.07829482853412628, "policy/entropy_avg": 0.19235610961914062, "val/ratio": 0.9867407083511353, "val/ratio_var": 7.090938743203878e-05, "val/num_eos_tokens": 147, "lr": 6.112604669781572e-06, "episode": 9072, "epoch": 0.4581818181818182, "step": 567}, {"eps": 0, "objective/kl": 28.36505889892578, "objective/entropy": -65.55770874023438, "objective/non_score_reward": -1.418252944946289, "objective/rlhf_reward": 13.370809555053711, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0009267916902899742, "policy/clipfrac_avg": 0.013094292022287846, "loss/policy_avg": -0.19695457816123962, "loss/value_avg": 0.6798470616340637, "val/clipfrac_avg": 0.11755470186471939, "policy/entropy_avg": 0.15218353271484375, "val/ratio": 0.9906548261642456, "val/ratio_var": 8.349393465323374e-05, "val/num_eos_tokens": 145, "lr": 6.099578494275883e-06, "episode": 9088, "epoch": 0.458989898989899, "step": 568}, {"eps": 0, "objective/kl": 25.735273361206055, "objective/entropy": -75.15937042236328, "objective/non_score_reward": -1.2867636680603027, "objective/rlhf_reward": 12.408548355102539, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0009919216390699148, "policy/clipfrac_avg": 0.00809397827833891, "loss/policy_avg": 0.03359239920973778, "loss/value_avg": 0.5841665863990784, "val/clipfrac_avg": 0.11711117625236511, "policy/entropy_avg": 0.15917348861694336, "val/ratio": 0.9905821084976196, "val/ratio_var": 5.5863049055915326e-05, "val/num_eos_tokens": 421, "lr": 6.0865444716359745e-06, "episode": 9104, "epoch": 0.4597979797979798, "step": 569}, {"eps": 0, "objective/kl": 35.65166473388672, "objective/entropy": -101.9668197631836, "objective/non_score_reward": -1.782583236694336, "objective/rlhf_reward": 13.006479263305664, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0008678468875586987, "policy/clipfrac_avg": 0.009188210591673851, "loss/policy_avg": -0.1148596853017807, "loss/value_avg": 0.9369633197784424, "val/clipfrac_avg": 0.04170583188533783, "policy/entropy_avg": 0.1626424789428711, "val/ratio": 0.9893851280212402, "val/ratio_var": 6.942804611753672e-05, "val/num_eos_tokens": 411, "lr": 6.073502694879059e-06, "episode": 9120, "epoch": 0.46060606060606063, "step": 570}, {"eps": 0, "objective/kl": 34.878883361816406, "objective/entropy": -29.416015625, "objective/non_score_reward": -1.7439441680908203, "objective/rlhf_reward": 13.38105583190918, "objective/scores": 15.125, "policy/approxkl_avg": 0.0009020785801112652, "policy/clipfrac_avg": 0.005981042981147766, "loss/policy_avg": 0.03529290482401848, "loss/value_avg": 0.6290674209594727, "val/clipfrac_avg": 0.2255920171737671, "policy/entropy_avg": 0.17604637145996094, "val/ratio": 0.9879002571105957, "val/ratio_var": 6.197366747073829e-05, "val/num_eos_tokens": 197, "lr": 6.060453257077686e-06, "episode": 9136, "epoch": 0.4614141414141414, "step": 571}, {"eps": 0, "objective/kl": 32.74857711791992, "objective/entropy": -94.31534576416016, "objective/non_score_reward": -1.6374289989471436, "objective/rlhf_reward": 11.651633262634277, "objective/scores": 13.3125, "policy/approxkl_avg": 0.0009000588906928897, "policy/clipfrac_avg": 0.006325645837932825, "loss/policy_avg": 0.07575584203004837, "loss/value_avg": 0.48475825786590576, "val/clipfrac_avg": 0.1001669317483902, "policy/entropy_avg": 0.16740036010742188, "val/ratio": 0.9889869689941406, "val/ratio_var": 4.305645416025072e-05, "val/num_eos_tokens": 374, "lr": 6.047396251359073e-06, "episode": 9152, "epoch": 0.4622222222222222, "step": 572}, {"eps": 0, "objective/kl": 32.949615478515625, "objective/entropy": -134.7626495361328, "objective/non_score_reward": -1.6474809646606445, "objective/rlhf_reward": 13.688455581665039, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0008799454662948847, "policy/clipfrac_avg": 0.014949589036405087, "loss/policy_avg": 0.06133081763982773, "loss/value_avg": 0.6095678806304932, "val/clipfrac_avg": 0.0686127319931984, "policy/entropy_avg": 0.12122154235839844, "val/ratio": 0.9921667575836182, "val/ratio_var": 3.237271084799431e-05, "val/num_eos_tokens": 434, "lr": 6.034331770904455e-06, "episode": 9168, "epoch": 0.463030303030303, "step": 573}, {"eps": 0, "objective/kl": 35.466163635253906, "objective/entropy": -40.367774963378906, "objective/non_score_reward": -1.773308277130127, "objective/rlhf_reward": 11.523567199707031, "objective/scores": 13.3125, "policy/approxkl_avg": 0.001205465872772038, "policy/clipfrac_avg": 0.008106344379484653, "loss/policy_avg": 0.0019814386032521725, "loss/value_avg": 0.4525398015975952, "val/clipfrac_avg": 0.13928186893463135, "policy/entropy_avg": 0.2171649932861328, "val/ratio": 0.9869014024734497, "val/ratio_var": 5.974850137135945e-05, "val/num_eos_tokens": 19, "lr": 6.0212599089484026e-06, "episode": 9184, "epoch": 0.4638383838383838, "step": 574}, {"eps": 0, "objective/kl": 33.518157958984375, "objective/entropy": -219.04417419433594, "objective/non_score_reward": -1.675907850265503, "objective/rlhf_reward": 12.652216911315918, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0009288907749578357, "policy/clipfrac_avg": 0.014125748537480831, "loss/policy_avg": -0.14026030898094177, "loss/value_avg": 0.2715468406677246, "val/clipfrac_avg": 0.09028111398220062, "policy/entropy_avg": 0.12938570976257324, "val/ratio": 0.9909526705741882, "val/ratio_var": 6.470608786912635e-05, "val/num_eos_tokens": 30, "lr": 6.008180758778167e-06, "episode": 9200, "epoch": 0.46464646464646464, "step": 575}, {"eps": 0, "objective/kl": 33.288169860839844, "objective/entropy": -93.06488037109375, "objective/non_score_reward": -1.6644084453582764, "objective/rlhf_reward": 12.046529769897461, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0009613010915927589, "policy/clipfrac_avg": 0.006700174883008003, "loss/policy_avg": 0.12112675607204437, "loss/value_avg": 0.6767160892486572, "val/clipfrac_avg": 0.12174967676401138, "policy/entropy_avg": 0.17998027801513672, "val/ratio": 0.988273561000824, "val/ratio_var": 8.490013715345412e-05, "val/num_eos_tokens": 34, "lr": 5.9950944137330125e-06, "episode": 9216, "epoch": 0.46545454545454545, "step": 576}, {"eps": 0, "objective/kl": 28.850629806518555, "objective/entropy": -71.49658966064453, "objective/non_score_reward": -1.4425315856933594, "objective/rlhf_reward": 12.59653091430664, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0009972681291401386, "policy/clipfrac_avg": 0.009436940774321556, "loss/policy_avg": -0.041956715285778046, "loss/value_avg": 0.5106937885284424, "val/clipfrac_avg": 0.1011686772108078, "policy/entropy_avg": 0.1730337142944336, "val/ratio": 0.9901687502861023, "val/ratio_var": 5.90443960390985e-05, "val/num_eos_tokens": 21, "lr": 5.982000967203548e-06, "episode": 9232, "epoch": 0.46626262626262627, "step": 577}, {"eps": 0, "objective/kl": 33.98302459716797, "objective/entropy": -94.11358642578125, "objective/non_score_reward": -1.6991510391235352, "objective/rlhf_reward": 13.886785507202148, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0009757350198924541, "policy/clipfrac_avg": 0.009374083951115608, "loss/policy_avg": -0.12406820058822632, "loss/value_avg": 0.5695918798446655, "val/clipfrac_avg": 0.0964265689253807, "policy/entropy_avg": 0.1509075164794922, "val/ratio": 0.9889683723449707, "val/ratio_var": 6.234552711248398e-05, "val/num_eos_tokens": 261, "lr": 5.968900512631065e-06, "episode": 9248, "epoch": 0.4670707070707071, "step": 578}, {"eps": 0, "objective/kl": 40.66157150268555, "objective/entropy": -60.78998565673828, "objective/non_score_reward": -2.033078670501709, "objective/rlhf_reward": 13.380983352661133, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0010061386274173856, "policy/clipfrac_avg": 0.006983610801398754, "loss/policy_avg": -0.012504605576395988, "loss/value_avg": 0.16729792952537537, "val/clipfrac_avg": 0.14943929016590118, "policy/entropy_avg": 0.18235397338867188, "val/ratio": 0.9873251914978027, "val/ratio_var": 3.270859451731667e-05, "val/num_eos_tokens": 86, "lr": 5.955793143506863e-06, "episode": 9264, "epoch": 0.4678787878787879, "step": 579}, {"eps": 0, "objective/kl": 44.126407623291016, "objective/entropy": -80.41490173339844, "objective/non_score_reward": -2.206320285797119, "objective/rlhf_reward": 11.426491737365723, "objective/scores": 13.625, "policy/approxkl_avg": 0.0009176756720989943, "policy/clipfrac_avg": 0.005069363862276077, "loss/policy_avg": -0.05700581520795822, "loss/value_avg": 0.6241277456283569, "val/clipfrac_avg": 0.18845900893211365, "policy/entropy_avg": 0.17818450927734375, "val/ratio": 0.9875844717025757, "val/ratio_var": 7.17366929166019e-05, "val/num_eos_tokens": 51, "lr": 5.94267895337159e-06, "episode": 9280, "epoch": 0.4686868686868687, "step": 580}, {"eps": 0, "objective/kl": 37.18535614013672, "objective/entropy": -76.93428039550781, "objective/non_score_reward": -1.8592679500579834, "objective/rlhf_reward": 12.875106811523438, "objective/scores": 14.75, "policy/approxkl_avg": 0.0009798705577850342, "policy/clipfrac_avg": 0.007772150449454784, "loss/policy_avg": -0.06188559532165527, "loss/value_avg": 0.3465093970298767, "val/clipfrac_avg": 0.061571549624204636, "policy/entropy_avg": 0.1757030487060547, "val/ratio": 0.988136887550354, "val/ratio_var": 4.800504757440649e-05, "val/num_eos_tokens": 158, "lr": 5.929558035814574e-06, "episode": 9296, "epoch": 0.4694949494949495, "step": 581}, {"eps": 0, "objective/kl": 39.926918029785156, "objective/entropy": -64.74710845947266, "objective/non_score_reward": -1.996346116065979, "objective/rlhf_reward": 12.659903526306152, "objective/scores": 14.625, "policy/approxkl_avg": 0.000917055644094944, "policy/clipfrac_avg": 0.007335194386541843, "loss/policy_avg": 0.13406801223754883, "loss/value_avg": 0.6718024611473083, "val/clipfrac_avg": 0.1007988452911377, "policy/entropy_avg": 0.18127059936523438, "val/ratio": 0.9881880283355713, "val/ratio_var": 5.9562738897511736e-05, "val/num_eos_tokens": 383, "lr": 5.916430484473149e-06, "episode": 9312, "epoch": 0.4703030303030303, "step": 582}, {"eps": 0, "objective/kl": 32.04585647583008, "objective/entropy": -120.16492462158203, "objective/non_score_reward": -1.602292776107788, "objective/rlhf_reward": 13.272706985473633, "objective/scores": 14.875, "policy/approxkl_avg": 0.0009271152084693313, "policy/clipfrac_avg": 0.0157107375562191, "loss/policy_avg": 0.3530195653438568, "loss/value_avg": 0.4071064591407776, "val/clipfrac_avg": 0.041299499571323395, "policy/entropy_avg": 0.14319610595703125, "val/ratio": 0.9893313646316528, "val/ratio_var": 5.184482506592758e-05, "val/num_eos_tokens": 319, "lr": 5.903296393031996e-06, "episode": 9328, "epoch": 0.4711111111111111, "step": 583}, {"eps": 0, "objective/kl": 37.35072326660156, "objective/entropy": -21.60655975341797, "objective/non_score_reward": -1.8675363063812256, "objective/rlhf_reward": 13.484025955200195, "objective/scores": 15.375, "policy/approxkl_avg": 0.001100124092772603, "policy/clipfrac_avg": 0.010222227312624454, "loss/policy_avg": -0.03923230990767479, "loss/value_avg": 0.5645366907119751, "val/clipfrac_avg": 0.09733504056930542, "policy/entropy_avg": 0.17325496673583984, "val/ratio": 0.9891010522842407, "val/ratio_var": 5.214220436755568e-05, "val/num_eos_tokens": 8, "lr": 5.890155855222463e-06, "episode": 9344, "epoch": 0.4719191919191919, "step": 584}, {"eps": 0, "objective/kl": 43.35786437988281, "objective/entropy": -169.64785766601562, "objective/non_score_reward": -2.167893171310425, "objective/rlhf_reward": 12.074295043945312, "objective/scores": 14.25, "policy/approxkl_avg": 0.0007353628752753139, "policy/clipfrac_avg": 0.0040179151110351086, "loss/policy_avg": -0.27854421734809875, "loss/value_avg": 0.583124577999115, "val/clipfrac_avg": 0.21491241455078125, "policy/entropy_avg": 0.13879728317260742, "val/ratio": 0.990776777267456, "val/ratio_var": 4.720980359707028e-05, "val/num_eos_tokens": 92, "lr": 5.877008964821909e-06, "episode": 9360, "epoch": 0.4727272727272727, "step": 585}, {"eps": 0, "objective/kl": 49.1492919921875, "objective/entropy": -151.75347900390625, "objective/non_score_reward": -2.4574646949768066, "objective/rlhf_reward": 11.222222328186035, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0009073680848814547, "policy/clipfrac_avg": 0.00646246736869216, "loss/policy_avg": 0.051072392612695694, "loss/value_avg": 0.36724406480789185, "val/clipfrac_avg": 0.09604115784168243, "policy/entropy_avg": 0.1743144989013672, "val/ratio": 0.9888123273849487, "val/ratio_var": 3.67878055840265e-05, "val/num_eos_tokens": 162, "lr": 5.863855815653022e-06, "episode": 9376, "epoch": 0.47353535353535353, "step": 586}, {"eps": 0, "objective/kl": 36.873565673828125, "objective/entropy": -66.69998931884766, "objective/non_score_reward": -1.8436782360076904, "objective/rlhf_reward": 12.39850902557373, "objective/scores": 14.25, "policy/approxkl_avg": 0.0008563040173612535, "policy/clipfrac_avg": 0.014813702553510666, "loss/policy_avg": -0.07891325652599335, "loss/value_avg": 0.20245110988616943, "val/clipfrac_avg": 0.04891027510166168, "policy/entropy_avg": 0.1890716552734375, "val/ratio": 0.9900521039962769, "val/ratio_var": 4.2384406697237864e-05, "val/num_eos_tokens": 361, "lr": 5.850696501583164e-06, "episode": 9392, "epoch": 0.47434343434343434, "step": 587}, {"eps": 0, "objective/kl": 37.953643798828125, "objective/entropy": -44.46873092651367, "objective/non_score_reward": -1.8976821899414062, "objective/rlhf_reward": 13.446067810058594, "objective/scores": 15.375, "policy/approxkl_avg": 0.0010375550482422113, "policy/clipfrac_avg": 0.006371880881488323, "loss/policy_avg": 0.05903597176074982, "loss/value_avg": 0.44623643159866333, "val/clipfrac_avg": 0.1483321189880371, "policy/entropy_avg": 0.19812297821044922, "val/ratio": 0.9867931604385376, "val/ratio_var": 4.73033323942218e-05, "val/num_eos_tokens": 409, "lr": 5.837531116523683e-06, "episode": 9408, "epoch": 0.47515151515151516, "step": 588}, {"eps": 0, "objective/kl": 38.770042419433594, "objective/entropy": -42.065185546875, "objective/non_score_reward": -1.9385021924972534, "objective/rlhf_reward": 12.967748641967773, "objective/scores": 14.875, "policy/approxkl_avg": 0.0011438718065619469, "policy/clipfrac_avg": 0.010345622897148132, "loss/policy_avg": 0.10919662564992905, "loss/value_avg": 0.3661342263221741, "val/clipfrac_avg": 0.05800151452422142, "policy/entropy_avg": 0.20827293395996094, "val/ratio": 0.9861246347427368, "val/ratio_var": 5.3233892685966566e-05, "val/num_eos_tokens": 131, "lr": 5.8243597544292585e-06, "episode": 9424, "epoch": 0.47595959595959597, "step": 589}, {"eps": 0, "objective/kl": 43.380714416503906, "objective/entropy": -48.181251525878906, "objective/non_score_reward": -2.1690359115600586, "objective/rlhf_reward": 15.260650634765625, "objective/scores": 17.375, "policy/approxkl_avg": 0.0009571064729243517, "policy/clipfrac_avg": 0.007153085432946682, "loss/policy_avg": -0.10685920715332031, "loss/value_avg": 0.6044321656227112, "val/clipfrac_avg": 0.18321609497070312, "policy/entropy_avg": 0.1891775131225586, "val/ratio": 0.9876303672790527, "val/ratio_var": 7.049312989693135e-05, "val/num_eos_tokens": 322, "lr": 5.811182509297224e-06, "episode": 9440, "epoch": 0.4767676767676768, "step": 590}, {"eps": 0, "objective/kl": 38.937896728515625, "objective/entropy": -93.8370361328125, "objective/non_score_reward": -1.946894884109497, "objective/rlhf_reward": 13.224980354309082, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0009119432652369142, "policy/clipfrac_avg": 0.010985877364873886, "loss/policy_avg": -0.14776897430419922, "loss/value_avg": 0.4778248965740204, "val/clipfrac_avg": 0.0706997662782669, "policy/entropy_avg": 0.17043495178222656, "val/ratio": 0.9888738989830017, "val/ratio_var": 5.0667145842453465e-05, "val/num_eos_tokens": 209, "lr": 5.797999475166897e-06, "episode": 9456, "epoch": 0.4775757575757576, "step": 591}, {"eps": 0, "objective/kl": 35.89628601074219, "objective/entropy": -33.68360900878906, "objective/non_score_reward": -1.794814109802246, "objective/rlhf_reward": 12.595810890197754, "objective/scores": 14.375, "policy/approxkl_avg": 0.0008912064367905259, "policy/clipfrac_avg": 0.011523826979100704, "loss/policy_avg": 0.14408515393733978, "loss/value_avg": 0.5567938089370728, "val/clipfrac_avg": 0.15322092175483704, "policy/entropy_avg": 0.13869476318359375, "val/ratio": 0.9900076985359192, "val/ratio_var": 2.749385566858109e-05, "val/num_eos_tokens": 79, "lr": 5.784810746118907e-06, "episode": 9472, "epoch": 0.4783838383838384, "step": 592}, {"eps": 0, "objective/kl": 34.06424331665039, "objective/entropy": -79.10285949707031, "objective/non_score_reward": -1.7032122611999512, "objective/rlhf_reward": 12.25772476196289, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0009309371234849095, "policy/clipfrac_avg": 0.009452100843191147, "loss/policy_avg": 0.06612087786197662, "loss/value_avg": 0.5821722149848938, "val/clipfrac_avg": 0.1111634224653244, "policy/entropy_avg": 0.1573638916015625, "val/ratio": 0.9895343780517578, "val/ratio_var": 4.007742609246634e-05, "val/num_eos_tokens": 63, "lr": 5.77161641627453e-06, "episode": 9488, "epoch": 0.47919191919191917, "step": 593}, {"eps": 0, "objective/kl": 44.3161506652832, "objective/entropy": -173.3606719970703, "objective/non_score_reward": -2.2158076763153076, "objective/rlhf_reward": 11.315442085266113, "objective/scores": 13.5, "policy/approxkl_avg": 0.0008013836340978742, "policy/clipfrac_avg": 0.009280474856495857, "loss/policy_avg": -0.0943286195397377, "loss/value_avg": 0.2797660827636719, "val/clipfrac_avg": 0.04634127765893936, "policy/entropy_avg": 0.15043258666992188, "val/ratio": 0.9904316067695618, "val/ratio_var": 5.512699135579169e-05, "val/num_eos_tokens": 439, "lr": 5.7584165797950055e-06, "episode": 9504, "epoch": 0.48, "step": 594}, {"eps": 0, "objective/kl": 39.01966094970703, "objective/entropy": -66.67430877685547, "objective/non_score_reward": -1.9509832859039307, "objective/rlhf_reward": 14.345891952514648, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009961356408894062, "policy/clipfrac_avg": 0.006623497232794762, "loss/policy_avg": -0.17459821701049805, "loss/value_avg": 0.568943977355957, "val/clipfrac_avg": 0.09633146226406097, "policy/entropy_avg": 0.17397308349609375, "val/ratio": 0.988785982131958, "val/ratio_var": 4.24717363785021e-05, "val/num_eos_tokens": 305, "lr": 5.745211330880872e-06, "episode": 9520, "epoch": 0.4808080808080808, "step": 595}, {"eps": 0, "objective/kl": 36.13384246826172, "objective/entropy": -38.30708312988281, "objective/non_score_reward": -1.8066922426223755, "objective/rlhf_reward": 14.466745376586914, "objective/scores": 16.25, "policy/approxkl_avg": 0.000953501439653337, "policy/clipfrac_avg": 0.005453229416161776, "loss/policy_avg": -0.06750007718801498, "loss/value_avg": 0.25791820883750916, "val/clipfrac_avg": 0.08646031469106674, "policy/entropy_avg": 0.17740631103515625, "val/ratio": 0.9876688718795776, "val/ratio_var": 3.322769043734297e-05, "val/num_eos_tokens": 232, "lr": 5.732000763771301e-06, "episode": 9536, "epoch": 0.4816161616161616, "step": 596}, {"eps": 0, "objective/kl": 25.864574432373047, "objective/entropy": -108.50715637207031, "objective/non_score_reward": -1.2932287454605103, "objective/rlhf_reward": 12.941146850585938, "objective/scores": 14.25, "policy/approxkl_avg": 0.0007267807959578931, "policy/clipfrac_avg": 0.024973273277282715, "loss/policy_avg": 0.0824112594127655, "loss/value_avg": 1.29659903049469, "val/clipfrac_avg": 0.035626426339149475, "policy/entropy_avg": 0.12932777404785156, "val/ratio": 0.9911180734634399, "val/ratio_var": 5.246651198831387e-05, "val/num_eos_tokens": 90, "lr": 5.71878497274341e-06, "episode": 9552, "epoch": 0.4824242424242424, "step": 597}, {"eps": 0, "objective/kl": 42.281578063964844, "objective/entropy": -73.0534896850586, "objective/non_score_reward": -2.114078998565674, "objective/rlhf_reward": 14.260921478271484, "objective/scores": 16.375, "policy/approxkl_avg": 0.0010907119140028954, "policy/clipfrac_avg": 0.007879609242081642, "loss/policy_avg": -0.036184508353471756, "loss/value_avg": 0.2775241732597351, "val/clipfrac_avg": 0.0538523867726326, "policy/entropy_avg": 0.18637657165527344, "val/ratio": 0.9873398542404175, "val/ratio_var": 5.951778803137131e-05, "val/num_eos_tokens": 278, "lr": 5.705564052111597e-06, "episode": 9568, "epoch": 0.48323232323232324, "step": 598}, {"eps": 0, "objective/kl": 36.109771728515625, "objective/entropy": -93.86756896972656, "objective/non_score_reward": -1.8054884672164917, "objective/rlhf_reward": 12.616386413574219, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0010626658331602812, "policy/clipfrac_avg": 0.006483650300651789, "loss/policy_avg": -0.09331340342760086, "loss/value_avg": 0.4313856363296509, "val/clipfrac_avg": 0.053107958287000656, "policy/entropy_avg": 0.15071487426757812, "val/ratio": 0.9895498752593994, "val/ratio_var": 5.324727317201905e-05, "val/num_eos_tokens": 201, "lr": 5.69233809622687e-06, "episode": 9584, "epoch": 0.48404040404040405, "step": 599}, {"eps": 0, "objective/kl": 45.25263977050781, "objective/entropy": -186.87242126464844, "objective/non_score_reward": -2.262631893157959, "objective/rlhf_reward": 13.534242630004883, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0007487519760616124, "policy/clipfrac_avg": 0.002827760763466358, "loss/policy_avg": -0.1564587652683258, "loss/value_avg": 0.3211984634399414, "val/clipfrac_avg": 0.08722794055938721, "policy/entropy_avg": 0.1533966064453125, "val/ratio": 0.9899605512619019, "val/ratio_var": 3.863383972202428e-05, "val/num_eos_tokens": 503, "lr": 5.679107199476174e-06, "episode": 9600, "epoch": 0.48484848484848486, "step": 600}, {"eps": 0, "objective/kl": 41.667076110839844, "objective/entropy": -43.572662353515625, "objective/non_score_reward": -2.0833539962768555, "objective/rlhf_reward": 12.338520050048828, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0012729446170851588, "policy/clipfrac_avg": 0.015199655666947365, "loss/policy_avg": 0.2646865248680115, "loss/value_avg": 0.9588360786437988, "val/clipfrac_avg": 0.20558898150920868, "policy/entropy_avg": 0.1835641860961914, "val/ratio": 0.9891852140426636, "val/ratio_var": 8.858136425260454e-05, "val/num_eos_tokens": 18, "lr": 5.665871456281706e-06, "episode": 9616, "epoch": 0.4856565656565657, "step": 601}, {"eps": 0, "objective/kl": 40.64360809326172, "objective/entropy": -80.89665985107422, "objective/non_score_reward": -2.0321803092956543, "objective/rlhf_reward": 13.131881713867188, "objective/scores": 15.1875, "policy/approxkl_avg": 0.000914716743864119, "policy/clipfrac_avg": 0.0072946203872561455, "loss/policy_avg": 0.018315471708774567, "loss/value_avg": 0.5322003364562988, "val/clipfrac_avg": 0.08266787230968475, "policy/entropy_avg": 0.17550897598266602, "val/ratio": 0.9880762696266174, "val/ratio_var": 7.163242844399065e-05, "val/num_eos_tokens": 181, "lr": 5.65263096110026e-06, "episode": 9632, "epoch": 0.4864646464646465, "step": 602}, {"eps": 0, "objective/kl": 37.012176513671875, "objective/entropy": -56.91044616699219, "objective/non_score_reward": -1.8506088256835938, "objective/rlhf_reward": 13.821266174316406, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008462531259283423, "policy/clipfrac_avg": 0.003934571985155344, "loss/policy_avg": -0.22014892101287842, "loss/value_avg": 0.27795735001564026, "val/clipfrac_avg": 0.07405910640954971, "policy/entropy_avg": 0.15899276733398438, "val/ratio": 0.9893808960914612, "val/ratio_var": 5.0625403673620895e-05, "val/num_eos_tokens": 119, "lr": 5.6393858084225305e-06, "episode": 9648, "epoch": 0.48727272727272725, "step": 603}, {"eps": 0, "objective/kl": 45.011924743652344, "objective/entropy": 0.5770502090454102, "objective/non_score_reward": -2.250596046447754, "objective/rlhf_reward": 14.39002799987793, "objective/scores": 16.625, "policy/approxkl_avg": 0.0011445335112512112, "policy/clipfrac_avg": 0.004843855742365122, "loss/policy_avg": -0.13129989802837372, "loss/value_avg": 0.290170818567276, "val/clipfrac_avg": 0.053368281573057175, "policy/entropy_avg": 0.21940231323242188, "val/ratio": 0.9853905439376831, "val/ratio_var": 4.1561663238098845e-05, "val/num_eos_tokens": 82, "lr": 5.626136092772463e-06, "episode": 9664, "epoch": 0.48808080808080806, "step": 604}, {"eps": 0, "objective/kl": 29.19556999206543, "objective/entropy": -104.7314682006836, "objective/non_score_reward": -1.4597785472869873, "objective/rlhf_reward": 13.54022216796875, "objective/scores": 15.0, "policy/approxkl_avg": 0.0007493078010156751, "policy/clipfrac_avg": 0.005996751599013805, "loss/policy_avg": -0.15523457527160645, "loss/value_avg": 1.380023717880249, "val/clipfrac_avg": 0.11536569148302078, "policy/entropy_avg": 0.14516448974609375, "val/ratio": 0.9902224540710449, "val/ratio_var": 5.3489617130253464e-05, "val/num_eos_tokens": 465, "lr": 5.612881908706555e-06, "episode": 9680, "epoch": 0.4888888888888889, "step": 605}, {"eps": 0, "objective/kl": 39.36458969116211, "objective/entropy": -157.1469268798828, "objective/non_score_reward": -1.9682295322418213, "objective/rlhf_reward": 12.363801956176758, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0008582321461290121, "policy/clipfrac_avg": 0.010594002902507782, "loss/policy_avg": -0.27331143617630005, "loss/value_avg": 1.16130530834198, "val/clipfrac_avg": 0.0660855621099472, "policy/entropy_avg": 0.14953947067260742, "val/ratio": 0.9899133443832397, "val/ratio_var": 4.759729199577123e-05, "val/num_eos_tokens": 412, "lr": 5.599623350813202e-06, "episode": 9696, "epoch": 0.4896969696969697, "step": 606}, {"eps": 0, "objective/kl": 33.51747131347656, "objective/entropy": -105.65791320800781, "objective/non_score_reward": -1.6758736371994019, "objective/rlhf_reward": 13.281157493591309, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0008841350791044533, "policy/clipfrac_avg": 0.0069657666608691216, "loss/policy_avg": -0.19870899617671967, "loss/value_avg": 0.37283676862716675, "val/clipfrac_avg": 0.0555139034986496, "policy/entropy_avg": 0.15077972412109375, "val/ratio": 0.9894367456436157, "val/ratio_var": 7.082260708557442e-05, "val/num_eos_tokens": 146, "lr": 5.586360513712011e-06, "episode": 9712, "epoch": 0.4905050505050505, "step": 607}, {"eps": 0, "objective/kl": 36.81121063232422, "objective/entropy": -30.997875213623047, "objective/non_score_reward": -1.8405605554580688, "objective/rlhf_reward": 12.971939086914062, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0010020849294960499, "policy/clipfrac_avg": 0.009872104972600937, "loss/policy_avg": 0.1218261867761612, "loss/value_avg": 0.6498777866363525, "val/clipfrac_avg": 0.09282970428466797, "policy/entropy_avg": 0.1610260009765625, "val/ratio": 0.987944483757019, "val/ratio_var": 3.414455568417907e-05, "val/num_eos_tokens": 288, "lr": 5.57309349205312e-06, "episode": 9728, "epoch": 0.4913131313131313, "step": 608}, {"eps": 0, "objective/kl": 36.83974838256836, "objective/entropy": -79.7080078125, "objective/non_score_reward": -1.8419876098632812, "objective/rlhf_reward": 11.478324890136719, "objective/scores": 13.3125, "policy/approxkl_avg": 0.000889924238435924, "policy/clipfrac_avg": 0.012620176188647747, "loss/policy_avg": -0.11768662929534912, "loss/value_avg": 0.7992165684700012, "val/clipfrac_avg": 0.05175953358411789, "policy/entropy_avg": 0.1525745391845703, "val/ratio": 0.9897374510765076, "val/ratio_var": 4.77236790175084e-05, "val/num_eos_tokens": 13, "lr": 5.559822380516539e-06, "episode": 9744, "epoch": 0.4921212121212121, "step": 609}, {"eps": 0, "objective/kl": 33.13558578491211, "objective/entropy": -32.14586639404297, "objective/non_score_reward": -1.656779408454895, "objective/rlhf_reward": 12.784626007080078, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0012670141877606511, "policy/clipfrac_avg": 0.00890309177339077, "loss/policy_avg": -0.2454073429107666, "loss/value_avg": 1.018176555633545, "val/clipfrac_avg": 0.12171462923288345, "policy/entropy_avg": 0.1815023422241211, "val/ratio": 0.986750602722168, "val/ratio_var": 5.6634431530255824e-05, "val/num_eos_tokens": 215, "lr": 5.546547273811465e-06, "episode": 9760, "epoch": 0.49292929292929294, "step": 610}, {"eps": 0, "objective/kl": 32.43745803833008, "objective/entropy": -21.851404190063477, "objective/non_score_reward": -1.6218730211257935, "objective/rlhf_reward": 15.729690551757812, "objective/scores": 17.375, "policy/approxkl_avg": 0.0015964140184223652, "policy/clipfrac_avg": 0.010030158795416355, "loss/policy_avg": -0.05234597623348236, "loss/value_avg": 0.9506372213363647, "val/clipfrac_avg": 0.19708402454853058, "policy/entropy_avg": 0.1926116943359375, "val/ratio": 0.9859472513198853, "val/ratio_var": 9.207271068589762e-05, "val/num_eos_tokens": 8, "lr": 5.533268266675601e-06, "episode": 9776, "epoch": 0.49373737373737375, "step": 611}, {"eps": 0, "objective/kl": 35.33883285522461, "objective/entropy": -77.98741149902344, "objective/non_score_reward": -1.7669416666030884, "objective/rlhf_reward": 14.93618392944336, "objective/scores": 16.75, "policy/approxkl_avg": 0.0007928519044071436, "policy/clipfrac_avg": 0.029583897441625595, "loss/policy_avg": -0.0021607354283332825, "loss/value_avg": 0.45648711919784546, "val/clipfrac_avg": 0.07284848392009735, "policy/entropy_avg": 0.1448807716369629, "val/ratio": 0.9906433820724487, "val/ratio_var": 5.276045703794807e-05, "val/num_eos_tokens": 735, "lr": 5.5199854538744905e-06, "episode": 9792, "epoch": 0.49454545454545457, "step": 612}, {"eps": 0, "objective/kl": 38.19306182861328, "objective/entropy": -163.5639190673828, "objective/non_score_reward": -1.9096529483795166, "objective/rlhf_reward": 11.535659790039062, "objective/scores": 13.4375, "policy/approxkl_avg": 0.000834503909572959, "policy/clipfrac_avg": 0.006998932920396328, "loss/policy_avg": -0.08518673479557037, "loss/value_avg": 0.6295963525772095, "val/clipfrac_avg": 0.0412946417927742, "policy/entropy_avg": 0.2186279296875, "val/ratio": 0.9895168542861938, "val/ratio_var": 4.984688348486088e-05, "val/num_eos_tokens": 393, "lr": 5.50669893020083e-06, "episode": 9808, "epoch": 0.4953535353535354, "step": 613}, {"eps": 0, "objective/kl": 36.03221893310547, "objective/entropy": -78.88749694824219, "objective/non_score_reward": -1.8016109466552734, "objective/rlhf_reward": 13.745264053344727, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0010543245589360595, "policy/clipfrac_avg": 0.0122576504945755, "loss/policy_avg": -0.09563255310058594, "loss/value_avg": 0.3395100235939026, "val/clipfrac_avg": 0.03431976959109306, "policy/entropy_avg": 0.17931032180786133, "val/ratio": 0.9878480434417725, "val/ratio_var": 7.280903082573786e-05, "val/num_eos_tokens": 348, "lr": 5.493408790473808e-06, "episode": 9824, "epoch": 0.49616161616161614, "step": 614}, {"eps": 0, "objective/kl": 29.166854858398438, "objective/entropy": -71.89592742919922, "objective/non_score_reward": -1.4583427906036377, "objective/rlhf_reward": 11.682282447814941, "objective/scores": 13.125, "policy/approxkl_avg": 0.0010009764228016138, "policy/clipfrac_avg": 0.009198974817991257, "loss/policy_avg": -0.011238239705562592, "loss/value_avg": 1.1844186782836914, "val/clipfrac_avg": 0.15483875572681427, "policy/entropy_avg": 0.1814279556274414, "val/ratio": 0.9910566806793213, "val/ratio_var": 0.00011891258327523246, "val/num_eos_tokens": 405, "lr": 5.480115129538409e-06, "episode": 9840, "epoch": 0.49696969696969695, "step": 615}, {"eps": 0, "objective/kl": 39.84773635864258, "objective/entropy": -75.9026870727539, "objective/non_score_reward": -1.992386817932129, "objective/rlhf_reward": 10.148238182067871, "objective/scores": 12.125, "policy/approxkl_avg": 0.0012564335484057665, "policy/clipfrac_avg": 0.009645252488553524, "loss/policy_avg": 0.07725506275892258, "loss/value_avg": 1.2075074911117554, "val/clipfrac_avg": 0.09601636976003647, "policy/entropy_avg": 0.19198989868164062, "val/ratio": 0.9873752593994141, "val/ratio_var": 8.283311763079837e-05, "val/num_eos_tokens": 432, "lr": 5.466818042264754e-06, "episode": 9856, "epoch": 0.49777777777777776, "step": 616}, {"eps": 0, "objective/kl": 39.151649475097656, "objective/entropy": -117.6833267211914, "objective/non_score_reward": -1.9575825929641724, "objective/rlhf_reward": 13.112730026245117, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0008085196604952216, "policy/clipfrac_avg": 0.01115940697491169, "loss/policy_avg": -0.12253916263580322, "loss/value_avg": 0.34518784284591675, "val/clipfrac_avg": 0.08139624446630478, "policy/entropy_avg": 0.16089963912963867, "val/ratio": 0.9896994829177856, "val/ratio_var": 3.962041228078306e-05, "val/num_eos_tokens": 740, "lr": 5.453517623547412e-06, "episode": 9872, "epoch": 0.4985858585858586, "step": 617}, {"eps": 0, "objective/kl": 27.95919418334961, "objective/entropy": -76.56217193603516, "objective/non_score_reward": -1.3979597091674805, "objective/rlhf_reward": 15.203603744506836, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008653640979900956, "policy/clipfrac_avg": 0.009502979926764965, "loss/policy_avg": 0.13594239950180054, "loss/value_avg": 0.6978110074996948, "val/clipfrac_avg": 0.03875832259654999, "policy/entropy_avg": 0.16449260711669922, "val/ratio": 0.9884722828865051, "val/ratio_var": 7.256936805788428e-05, "val/num_eos_tokens": 5, "lr": 5.440213968304728e-06, "episode": 9888, "epoch": 0.4993939393939394, "step": 618}, {"eps": 0, "objective/kl": 35.96702194213867, "objective/entropy": -51.29543685913086, "objective/non_score_reward": -1.7983510494232178, "objective/rlhf_reward": 13.365711212158203, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0010904096998274326, "policy/clipfrac_avg": 0.007610537111759186, "loss/policy_avg": 0.03163297474384308, "loss/value_avg": 0.5913329124450684, "val/clipfrac_avg": 0.1835818588733673, "policy/entropy_avg": 0.18178224563598633, "val/ratio": 0.9881883859634399, "val/ratio_var": 4.50633124273736e-05, "val/num_eos_tokens": 139, "lr": 5.426907171478143e-06, "episode": 9904, "epoch": 0.5002020202020202, "step": 619}, {"eps": 0, "objective/kl": 41.84601593017578, "objective/entropy": -40.40581130981445, "objective/non_score_reward": -2.0923008918762207, "objective/rlhf_reward": 14.431137084960938, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009933523833751678, "policy/clipfrac_avg": 0.004145262762904167, "loss/policy_avg": -0.018618369475007057, "loss/value_avg": 0.1497330665588379, "val/clipfrac_avg": 0.017992915585637093, "policy/entropy_avg": 0.1963958740234375, "val/ratio": 0.9876341819763184, "val/ratio_var": 2.160732947231736e-05, "val/num_eos_tokens": 503, "lr": 5.413597328031522e-06, "episode": 9920, "epoch": 0.501010101010101, "step": 620}, {"eps": 0, "objective/kl": 34.00743103027344, "objective/entropy": -250.99803161621094, "objective/non_score_reward": -1.7003716230392456, "objective/rlhf_reward": 13.354316711425781, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0008134699310176075, "policy/clipfrac_avg": 0.019926324486732483, "loss/policy_avg": 0.0749528706073761, "loss/value_avg": 0.6211892366409302, "val/clipfrac_avg": 0.060360997915267944, "policy/entropy_avg": 0.15630340576171875, "val/ratio": 0.9913326501846313, "val/ratio_var": 4.280313805793412e-05, "val/num_eos_tokens": 21, "lr": 5.4002845329504675e-06, "episode": 9936, "epoch": 0.5018181818181818, "step": 621}, {"eps": 0, "objective/kl": 22.701366424560547, "objective/entropy": -123.3650894165039, "objective/non_score_reward": -1.1350682973861694, "objective/rlhf_reward": 13.3727445602417, "objective/scores": 14.5, "policy/approxkl_avg": 0.0009935309644788504, "policy/clipfrac_avg": 0.008781377226114273, "loss/policy_avg": 0.36562579870224, "loss/value_avg": 1.0110520124435425, "val/clipfrac_avg": 0.14134380221366882, "policy/entropy_avg": 0.1174173355102539, "val/ratio": 0.9914640188217163, "val/ratio_var": 6.538646994158626e-05, "val/num_eos_tokens": 221, "lr": 5.386968881241649e-06, "episode": 9952, "epoch": 0.5026262626262626, "step": 622}, {"eps": 0, "objective/kl": 36.585201263427734, "objective/entropy": -120.33831787109375, "objective/non_score_reward": -1.8292601108551025, "objective/rlhf_reward": 12.944177627563477, "objective/scores": 14.75, "policy/approxkl_avg": 0.0015983078628778458, "policy/clipfrac_avg": 0.027648624032735825, "loss/policy_avg": 0.21662172675132751, "loss/value_avg": 0.46216171979904175, "val/clipfrac_avg": 0.1728610098361969, "policy/entropy_avg": 0.19037723541259766, "val/ratio": 0.9866871237754822, "val/ratio_var": 0.00018466412439011037, "val/num_eos_tokens": 333, "lr": 5.373650467932122e-06, "episode": 9968, "epoch": 0.5034343434343435, "step": 623}, {"eps": 0, "objective/kl": 34.64921951293945, "objective/entropy": -107.19503784179688, "objective/non_score_reward": -1.7324607372283936, "objective/rlhf_reward": 11.345664024353027, "objective/scores": 13.0625, "policy/approxkl_avg": 0.0009691518498584628, "policy/clipfrac_avg": 0.006136470474302769, "loss/policy_avg": 0.03029388189315796, "loss/value_avg": 0.5422893762588501, "val/clipfrac_avg": 0.06680267304182053, "policy/entropy_avg": 0.1900615692138672, "val/ratio": 0.988233208656311, "val/ratio_var": 7.25203353795223e-05, "val/num_eos_tokens": 650, "lr": 5.360329388068649e-06, "episode": 9984, "epoch": 0.5042424242424243, "step": 624}, {"eps": 0, "objective/kl": 37.40558624267578, "objective/entropy": -101.5595474243164, "objective/non_score_reward": -1.8702794313430786, "objective/rlhf_reward": 13.192220687866211, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0008968232432380319, "policy/clipfrac_avg": 0.014711007475852966, "loss/policy_avg": 0.2465195208787918, "loss/value_avg": 0.6872731447219849, "val/clipfrac_avg": 0.09929686039686203, "policy/entropy_avg": 0.14348411560058594, "val/ratio": 0.9888505935668945, "val/ratio_var": 4.519468348007649e-05, "val/num_eos_tokens": 428, "lr": 5.347005736717024e-06, "episode": 10000, "epoch": 0.5050505050505051, "step": 625}, {"eps": 0, "objective/kl": 39.13426971435547, "objective/entropy": -147.00888061523438, "objective/non_score_reward": -1.9567137956619263, "objective/rlhf_reward": 14.191723823547363, "objective/scores": 16.125, "policy/approxkl_avg": 0.0008501397678628564, "policy/clipfrac_avg": 0.00891679897904396, "loss/policy_avg": -0.09769874811172485, "loss/value_avg": 0.2699797451496124, "val/clipfrac_avg": 0.0778140276670456, "policy/entropy_avg": 0.19576263427734375, "val/ratio": 0.9883639812469482, "val/ratio_var": 7.028185063973069e-05, "val/num_eos_tokens": 156, "lr": 5.333679608961389e-06, "episode": 10016, "epoch": 0.5058585858585859, "step": 626}, {"eps": 0, "objective/kl": 31.409622192382812, "objective/entropy": -60.07448196411133, "objective/non_score_reward": -1.5704810619354248, "objective/rlhf_reward": 14.788894653320312, "objective/scores": 16.375, "policy/approxkl_avg": 0.000859480001963675, "policy/clipfrac_avg": 0.007050718180835247, "loss/policy_avg": -0.15752656757831573, "loss/value_avg": 1.4141845703125, "val/clipfrac_avg": 0.11736910045146942, "policy/entropy_avg": 0.1713123321533203, "val/ratio": 0.9884116649627686, "val/ratio_var": 6.11452414887026e-05, "val/num_eos_tokens": 235, "lr": 5.320351099903565e-06, "episode": 10032, "epoch": 0.5066666666666667, "step": 627}, {"eps": 0, "objective/kl": 32.27813720703125, "objective/entropy": -94.50772094726562, "objective/non_score_reward": -1.6139068603515625, "objective/rlhf_reward": 13.768905639648438, "objective/scores": 15.375, "policy/approxkl_avg": 0.0007259462727233768, "policy/clipfrac_avg": 0.005907295271754265, "loss/policy_avg": -0.3966858983039856, "loss/value_avg": 0.726300835609436, "val/clipfrac_avg": 0.2497040331363678, "policy/entropy_avg": 0.13324737548828125, "val/ratio": 0.9917147159576416, "val/ratio_var": 4.8239635361824185e-05, "val/num_eos_tokens": 195, "lr": 5.307020304662361e-06, "episode": 10048, "epoch": 0.5074747474747475, "step": 628}, {"eps": 0, "objective/kl": 30.607051849365234, "objective/entropy": -56.479736328125, "objective/non_score_reward": -1.5303525924682617, "objective/rlhf_reward": 12.360271453857422, "objective/scores": 13.875, "policy/approxkl_avg": 0.0010972167365252972, "policy/clipfrac_avg": 0.00503067672252655, "loss/policy_avg": 0.10847073793411255, "loss/value_avg": 1.0482819080352783, "val/clipfrac_avg": 0.18077512085437775, "policy/entropy_avg": 0.1625370979309082, "val/ratio": 0.9892880916595459, "val/ratio_var": 6.566623051185161e-05, "val/num_eos_tokens": 22, "lr": 5.293687318372907e-06, "episode": 10064, "epoch": 0.5082828282828283, "step": 629}, {"eps": 0, "objective/kl": 39.755985260009766, "objective/entropy": -135.9888916015625, "objective/non_score_reward": -1.9877994060516357, "objective/rlhf_reward": 12.520012855529785, "objective/scores": 14.5, "policy/approxkl_avg": 0.0009651440195739269, "policy/clipfrac_avg": 0.008618216961622238, "loss/policy_avg": -0.23807989060878754, "loss/value_avg": 0.4602620005607605, "val/clipfrac_avg": 0.08070136606693268, "policy/entropy_avg": 0.17929983139038086, "val/ratio": 0.9886597990989685, "val/ratio_var": 4.691245703725144e-05, "val/num_eos_tokens": 379, "lr": 5.2803522361859596e-06, "episode": 10080, "epoch": 0.509090909090909, "step": 630}, {"eps": 0, "objective/kl": 37.73855209350586, "objective/entropy": -94.9400634765625, "objective/non_score_reward": -1.886927604675293, "objective/rlhf_reward": 13.628697395324707, "objective/scores": 15.5, "policy/approxkl_avg": 0.0009247865527868271, "policy/clipfrac_avg": 0.009060590527951717, "loss/policy_avg": 0.23663482069969177, "loss/value_avg": 0.8216120004653931, "val/clipfrac_avg": 0.14852876961231232, "policy/entropy_avg": 0.16341590881347656, "val/ratio": 0.988681435585022, "val/ratio_var": 6.258696521399543e-05, "val/num_eos_tokens": 182, "lr": 5.267015153267246e-06, "episode": 10096, "epoch": 0.5098989898989899, "step": 631}, {"eps": 0, "objective/kl": 36.896602630615234, "objective/entropy": -80.35845947265625, "objective/non_score_reward": -1.8448302745819092, "objective/rlhf_reward": 11.233295440673828, "objective/scores": 13.0625, "policy/approxkl_avg": 0.0012214542366564274, "policy/clipfrac_avg": 0.006619518622756004, "loss/policy_avg": -0.08203352987766266, "loss/value_avg": 0.8431352376937866, "val/clipfrac_avg": 0.08380433917045593, "policy/entropy_avg": 0.19206619262695312, "val/ratio": 0.9882425665855408, "val/ratio_var": 0.00010104960529133677, "val/num_eos_tokens": 526, "lr": 5.25367616479676e-06, "episode": 10112, "epoch": 0.5107070707070707, "step": 632}, {"eps": 0, "objective/kl": 31.87903594970703, "objective/entropy": -83.84121704101562, "objective/non_score_reward": -1.59395170211792, "objective/rlhf_reward": 11.906047821044922, "objective/scores": 13.5, "policy/approxkl_avg": 0.00077263661660254, "policy/clipfrac_avg": 0.008104969747364521, "loss/policy_avg": -0.20128124952316284, "loss/value_avg": 0.36594098806381226, "val/clipfrac_avg": 0.10230521857738495, "policy/entropy_avg": 0.1480419635772705, "val/ratio": 0.9903359413146973, "val/ratio_var": 5.447642979561351e-05, "val/num_eos_tokens": 340, "lr": 5.240335365968104e-06, "episode": 10128, "epoch": 0.5115151515151515, "step": 633}, {"eps": 0, "objective/kl": 34.82407760620117, "objective/entropy": -79.21052551269531, "objective/non_score_reward": -1.741204023361206, "objective/rlhf_reward": 12.868170738220215, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009523687185719609, "policy/clipfrac_avg": 0.009445530362427235, "loss/policy_avg": 0.45129337906837463, "loss/value_avg": 1.256578803062439, "val/clipfrac_avg": 0.2403840720653534, "policy/entropy_avg": 0.18542003631591797, "val/ratio": 0.9878572821617126, "val/ratio_var": 5.5977081501623616e-05, "val/num_eos_tokens": 511, "lr": 5.226992851987793e-06, "episode": 10144, "epoch": 0.5123232323232323, "step": 634}, {"eps": 0, "objective/kl": 41.36100769042969, "objective/entropy": -60.33174133300781, "objective/non_score_reward": -2.0680503845214844, "objective/rlhf_reward": 13.408512115478516, "objective/scores": 15.5, "policy/approxkl_avg": 0.0012648601550608873, "policy/clipfrac_avg": 0.013032745569944382, "loss/policy_avg": -0.2209135740995407, "loss/value_avg": 0.4546852111816406, "val/clipfrac_avg": 0.0490388460457325, "policy/entropy_avg": 0.1959362030029297, "val/ratio": 0.9876245260238647, "val/ratio_var": 5.1731625717366114e-05, "val/num_eos_tokens": 286, "lr": 5.213648718074584e-06, "episode": 10160, "epoch": 0.5131313131313131, "step": 635}, {"eps": 0, "objective/kl": 30.6090030670166, "objective/entropy": -65.56263732910156, "objective/non_score_reward": -1.5304501056671143, "objective/rlhf_reward": 12.985175132751465, "objective/scores": 14.5, "policy/approxkl_avg": 0.0009048302308656275, "policy/clipfrac_avg": 0.009688157588243484, "loss/policy_avg": 0.30760645866394043, "loss/value_avg": 1.3313541412353516, "val/clipfrac_avg": 0.08400047570466995, "policy/entropy_avg": 0.16722917556762695, "val/ratio": 0.988949716091156, "val/ratio_var": 5.8918587455991656e-05, "val/num_eos_tokens": 173, "lr": 5.2003030594587964e-06, "episode": 10176, "epoch": 0.5139393939393939, "step": 636}, {"eps": 0, "objective/kl": 46.15885543823242, "objective/entropy": -102.1771240234375, "objective/non_score_reward": -2.3079428672790527, "objective/rlhf_reward": 11.965494155883789, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009750047465786338, "policy/clipfrac_avg": 0.008077681064605713, "loss/policy_avg": -0.03748584911227226, "loss/value_avg": 0.3042546808719635, "val/clipfrac_avg": 0.051075853407382965, "policy/entropy_avg": 0.168487548828125, "val/ratio": 0.98898845911026, "val/ratio_var": 4.223840733175166e-05, "val/num_eos_tokens": 307, "lr": 5.18695597138163e-06, "episode": 10192, "epoch": 0.5147474747474747, "step": 637}, {"eps": 0, "objective/kl": 37.841087341308594, "objective/entropy": -136.01560974121094, "objective/non_score_reward": -1.8920543193817139, "objective/rlhf_reward": 12.225133895874023, "objective/scores": 14.125, "policy/approxkl_avg": 0.001157400431111455, "policy/clipfrac_avg": 0.008751263841986656, "loss/policy_avg": -0.23826318979263306, "loss/value_avg": 0.7353537082672119, "val/clipfrac_avg": 0.0865342766046524, "policy/entropy_avg": 0.19327354431152344, "val/ratio": 0.9871768951416016, "val/ratio_var": 6.5818581788335e-05, "val/num_eos_tokens": 1837, "lr": 5.1736075490944815e-06, "episode": 10208, "epoch": 0.5155555555555555, "step": 638}, {"eps": 0, "objective/kl": 37.96650314331055, "objective/entropy": -36.917694091796875, "objective/non_score_reward": -1.8983252048492432, "objective/rlhf_reward": 14.390737533569336, "objective/scores": 16.25, "policy/approxkl_avg": 0.000980690703727305, "policy/clipfrac_avg": 0.007268393412232399, "loss/policy_avg": 0.015793576836586, "loss/value_avg": 0.3555375635623932, "val/clipfrac_avg": 0.04352998733520508, "policy/entropy_avg": 0.17595863342285156, "val/ratio": 0.9883407354354858, "val/ratio_var": 4.279622953617945e-05, "val/num_eos_tokens": 116, "lr": 5.160257887858278e-06, "episode": 10224, "epoch": 0.5163636363636364, "step": 639}, {"eps": 0, "objective/kl": 33.419410705566406, "objective/entropy": -150.77487182617188, "objective/non_score_reward": -1.6709705591201782, "objective/rlhf_reward": 12.969654083251953, "objective/scores": 14.625, "policy/approxkl_avg": 0.0007614089408889413, "policy/clipfrac_avg": 0.00750840874388814, "loss/policy_avg": -0.15323098003864288, "loss/value_avg": 0.3132088780403137, "val/clipfrac_avg": 0.06061185896396637, "policy/entropy_avg": 0.14741188287734985, "val/ratio": 0.9901533126831055, "val/ratio_var": 4.773136606672779e-05, "val/num_eos_tokens": 86, "lr": 5.146907082942778e-06, "episode": 10240, "epoch": 0.5171717171717172, "step": 640}, {"eps": 0, "objective/kl": 46.53107452392578, "objective/entropy": -184.5850067138672, "objective/non_score_reward": -2.3265538215637207, "objective/rlhf_reward": 13.353133201599121, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008069286122918129, "policy/clipfrac_avg": 0.004317488521337509, "loss/policy_avg": -0.40618008375167847, "loss/value_avg": 0.39824044704437256, "val/clipfrac_avg": 0.20341260731220245, "policy/entropy_avg": 0.15611839294433594, "val/ratio": 0.9897021055221558, "val/ratio_var": 4.78306865261402e-05, "val/num_eos_tokens": 56, "lr": 5.133555229625912e-06, "episode": 10256, "epoch": 0.517979797979798, "step": 641}, {"eps": 0, "objective/kl": 32.67811965942383, "objective/entropy": -92.79855346679688, "objective/non_score_reward": -1.6339060068130493, "objective/rlhf_reward": 11.483281135559082, "objective/scores": 13.125, "policy/approxkl_avg": 0.000959779426921159, "policy/clipfrac_avg": 0.013718648813664913, "loss/policy_avg": 0.35164520144462585, "loss/value_avg": 0.761013925075531, "val/clipfrac_avg": 0.10013719648122787, "policy/entropy_avg": 0.2091999053955078, "val/ratio": 0.9883617162704468, "val/ratio_var": 7.446538074873388e-05, "val/num_eos_tokens": 227, "lr": 5.120202423193085e-06, "episode": 10272, "epoch": 0.5187878787878788, "step": 642}, {"eps": 0, "objective/kl": 27.626182556152344, "objective/entropy": -61.3033447265625, "objective/non_score_reward": -1.381309151649475, "objective/rlhf_reward": 14.431190490722656, "objective/scores": 15.8125, "policy/approxkl_avg": 0.000927220331504941, "policy/clipfrac_avg": 0.01159143727272749, "loss/policy_avg": 0.17767910659313202, "loss/value_avg": 0.5378450155258179, "val/clipfrac_avg": 0.08490148931741714, "policy/entropy_avg": 0.17319107055664062, "val/ratio": 0.9895293712615967, "val/ratio_var": 5.0640650442801416e-05, "val/num_eos_tokens": 201, "lr": 5.106848758936508e-06, "episode": 10288, "epoch": 0.5195959595959596, "step": 643}, {"eps": 0, "objective/kl": 42.24378967285156, "objective/entropy": -100.7375717163086, "objective/non_score_reward": -2.112189292907715, "objective/rlhf_reward": 12.333123207092285, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0008687187219038606, "policy/clipfrac_avg": 0.005677542649209499, "loss/policy_avg": -0.004831582307815552, "loss/value_avg": 0.30399805307388306, "val/clipfrac_avg": 0.03101455792784691, "policy/entropy_avg": 0.17821407318115234, "val/ratio": 0.9884386658668518, "val/ratio_var": 4.8310637794202194e-05, "val/num_eos_tokens": 530, "lr": 5.093494332154511e-06, "episode": 10304, "epoch": 0.5204040404040404, "step": 644}, {"eps": 0, "objective/kl": 44.610504150390625, "objective/entropy": -77.07893371582031, "objective/non_score_reward": -2.230525255203247, "objective/rlhf_reward": 10.777286529541016, "objective/scores": 13.0, "policy/approxkl_avg": 0.00098347757011652, "policy/clipfrac_avg": 0.0057642534375190735, "loss/policy_avg": -0.1480829268693924, "loss/value_avg": 1.2215086221694946, "val/clipfrac_avg": 0.06691773980855942, "policy/entropy_avg": 0.20584464073181152, "val/ratio": 0.987593412399292, "val/ratio_var": 6.559812754858285e-05, "val/num_eos_tokens": 44, "lr": 5.080139238150869e-06, "episode": 10320, "epoch": 0.5212121212121212, "step": 645}, {"eps": 0, "objective/kl": 37.040985107421875, "objective/entropy": -102.19164276123047, "objective/non_score_reward": -1.8520495891571045, "objective/rlhf_reward": 15.030763626098633, "objective/scores": 16.875, "policy/approxkl_avg": 0.0008576101390644908, "policy/clipfrac_avg": 0.005198730155825615, "loss/policy_avg": 0.021136268973350525, "loss/value_avg": 0.24227744340896606, "val/clipfrac_avg": 0.023988839238882065, "policy/entropy_avg": 0.1648244857788086, "val/ratio": 0.988852858543396, "val/ratio_var": 3.1189680157694966e-05, "val/num_eos_tokens": 250, "lr": 5.066783572234115e-06, "episode": 10336, "epoch": 0.522020202020202, "step": 646}, {"eps": 0, "objective/kl": 39.353092193603516, "objective/entropy": -93.60134887695312, "objective/non_score_reward": -1.9676547050476074, "objective/rlhf_reward": 13.829219818115234, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0009072768152691424, "policy/clipfrac_avg": 0.010927075520157814, "loss/policy_avg": 0.1345120370388031, "loss/value_avg": 0.6760163307189941, "val/clipfrac_avg": 0.10246877372264862, "policy/entropy_avg": 0.16292381286621094, "val/ratio": 0.988797664642334, "val/ratio_var": 6.221264629857615e-05, "val/num_eos_tokens": 159, "lr": 5.053427429716867e-06, "episode": 10352, "epoch": 0.5228282828282829, "step": 647}, {"eps": 0, "objective/kl": 45.733909606933594, "objective/entropy": -218.74851989746094, "objective/non_score_reward": -2.2866954803466797, "objective/rlhf_reward": 13.22892951965332, "objective/scores": 15.5, "policy/approxkl_avg": 0.0007575824274681509, "policy/clipfrac_avg": 0.012265796773135662, "loss/policy_avg": -0.03434079885482788, "loss/value_avg": 0.26276272535324097, "val/clipfrac_avg": 0.09911301732063293, "policy/entropy_avg": 0.1510915756225586, "val/ratio": 0.9897732138633728, "val/ratio_var": 5.4691558034392074e-05, "val/num_eos_tokens": 287, "lr": 5.040070905915139e-06, "episode": 10368, "epoch": 0.5236363636363637, "step": 648}, {"eps": 0, "objective/kl": 38.24567794799805, "objective/entropy": -154.69427490234375, "objective/non_score_reward": -1.9122838973999023, "objective/rlhf_reward": 12.079903602600098, "objective/scores": 14.0, "policy/approxkl_avg": 0.0007898483891040087, "policy/clipfrac_avg": 0.004851070698350668, "loss/policy_avg": 0.17616848647594452, "loss/value_avg": 0.9873183965682983, "val/clipfrac_avg": 0.05088340863585472, "policy/entropy_avg": 0.18099498748779297, "val/ratio": 0.9899267554283142, "val/ratio_var": 4.2158419091720134e-05, "val/num_eos_tokens": 1593, "lr": 5.026714096147673e-06, "episode": 10384, "epoch": 0.5244444444444445, "step": 649}, {"eps": 0, "objective/kl": 27.840486526489258, "objective/entropy": -90.748779296875, "objective/non_score_reward": -1.392024278640747, "objective/rlhf_reward": 14.357975006103516, "objective/scores": 15.75, "policy/approxkl_avg": 0.0008530996856279671, "policy/clipfrac_avg": 0.005986053496599197, "loss/policy_avg": -0.10312466323375702, "loss/value_avg": 0.7348710298538208, "val/clipfrac_avg": 0.17565612494945526, "policy/entropy_avg": 0.14681434631347656, "val/ratio": 0.9898090362548828, "val/ratio_var": 6.2022045312915e-05, "val/num_eos_tokens": 591, "lr": 5.013357095735246e-06, "episode": 10400, "epoch": 0.5252525252525253, "step": 650}, {"eps": 0, "objective/kl": 36.00543975830078, "objective/entropy": -116.00996398925781, "objective/non_score_reward": -1.800271987915039, "objective/rlhf_reward": 13.379415512084961, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0008067564340308309, "policy/clipfrac_avg": 0.006692061200737953, "loss/policy_avg": -0.37787753343582153, "loss/value_avg": 0.4072280526161194, "val/clipfrac_avg": 0.12555305659770966, "policy/entropy_avg": 0.15956878662109375, "val/ratio": 0.9889513850212097, "val/ratio_var": 7.936693145893514e-05, "val/num_eos_tokens": 475, "lr": 5e-06, "episode": 10416, "epoch": 0.526060606060606, "step": 651}, {"eps": 0, "objective/kl": 35.3900260925293, "objective/entropy": -113.37627410888672, "objective/non_score_reward": -1.7695012092590332, "objective/rlhf_reward": 12.878936767578125, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009043759200721979, "policy/clipfrac_avg": 0.011335442773997784, "loss/policy_avg": -0.024874819442629814, "loss/value_avg": 0.30301523208618164, "val/clipfrac_avg": 0.05399475246667862, "policy/entropy_avg": 0.16724395751953125, "val/ratio": 0.989266037940979, "val/ratio_var": 4.7810990508878604e-05, "val/num_eos_tokens": 72, "lr": 4.986642904264754e-06, "episode": 10432, "epoch": 0.5268686868686868, "step": 652}, {"eps": 0, "objective/kl": 31.19072914123535, "objective/entropy": -97.41447448730469, "objective/non_score_reward": -1.5595364570617676, "objective/rlhf_reward": 14.838900566101074, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008020643144845963, "policy/clipfrac_avg": 0.003582795150578022, "loss/policy_avg": -0.4576267898082733, "loss/value_avg": 0.6310689449310303, "val/clipfrac_avg": 0.17964357137680054, "policy/entropy_avg": 0.16522979736328125, "val/ratio": 0.9892231225967407, "val/ratio_var": 5.0224600272485986e-05, "val/num_eos_tokens": 25, "lr": 4.973285903852329e-06, "episode": 10448, "epoch": 0.5276767676767676, "step": 653}, {"eps": 0, "objective/kl": 33.45720672607422, "objective/entropy": -157.48797607421875, "objective/non_score_reward": -1.6728602647781372, "objective/rlhf_reward": 12.967764854431152, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009867304470390081, "policy/clipfrac_avg": 0.008453142829239368, "loss/policy_avg": -0.1491837352514267, "loss/value_avg": 0.7328892946243286, "val/clipfrac_avg": 0.10647779703140259, "policy/entropy_avg": 0.1548633575439453, "val/ratio": 0.98896324634552, "val/ratio_var": 0.00010884697985602543, "val/num_eos_tokens": 1553, "lr": 4.959929094084862e-06, "episode": 10464, "epoch": 0.5284848484848484, "step": 654}, {"eps": 0, "objective/kl": 41.44767761230469, "objective/entropy": -151.52291870117188, "objective/non_score_reward": -2.0723838806152344, "objective/rlhf_reward": 11.599491119384766, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0010196359362453222, "policy/clipfrac_avg": 0.008509906008839607, "loss/policy_avg": -0.0017375200986862183, "loss/value_avg": 0.4749241769313812, "val/clipfrac_avg": 0.13483279943466187, "policy/entropy_avg": 0.17792892456054688, "val/ratio": 0.9885885715484619, "val/ratio_var": 3.869455758831464e-05, "val/num_eos_tokens": 723, "lr": 4.946572570283135e-06, "episode": 10480, "epoch": 0.5292929292929293, "step": 655}, {"eps": 0, "objective/kl": 37.215736389160156, "objective/entropy": -98.99279022216797, "objective/non_score_reward": -1.8607869148254395, "objective/rlhf_reward": 12.459526062011719, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0008932605269365013, "policy/clipfrac_avg": 0.01308157853782177, "loss/policy_avg": 0.30413296818733215, "loss/value_avg": 0.4696364402770996, "val/clipfrac_avg": 0.09068673849105835, "policy/entropy_avg": 0.16152191162109375, "val/ratio": 0.9888198971748352, "val/ratio_var": 6.496508285636082e-05, "val/num_eos_tokens": 495, "lr": 4.933216427765886e-06, "episode": 10496, "epoch": 0.5301010101010101, "step": 656}, {"eps": 0, "objective/kl": 38.032981872558594, "objective/entropy": -104.63330078125, "objective/non_score_reward": -1.9016492366790771, "objective/rlhf_reward": 12.348350524902344, "objective/scores": 14.25, "policy/approxkl_avg": 0.0007215352961793542, "policy/clipfrac_avg": 0.003871732857078314, "loss/policy_avg": 0.1066155657172203, "loss/value_avg": 1.9890694618225098, "val/clipfrac_avg": 0.17002993822097778, "policy/entropy_avg": 0.1489715576171875, "val/ratio": 0.9906018972396851, "val/ratio_var": 3.7619625800289214e-05, "val/num_eos_tokens": 9, "lr": 4.919860761849132e-06, "episode": 10512, "epoch": 0.5309090909090909, "step": 657}, {"eps": 0, "objective/kl": 36.28617858886719, "objective/entropy": -83.36598205566406, "objective/non_score_reward": -1.8143088817596436, "objective/rlhf_reward": 14.560690879821777, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008893824997358024, "policy/clipfrac_avg": 0.01114499568939209, "loss/policy_avg": 0.04127347841858864, "loss/value_avg": 0.2920118570327759, "val/clipfrac_avg": 0.032178279012441635, "policy/entropy_avg": 0.16919898986816406, "val/ratio": 0.9892511367797852, "val/ratio_var": 3.130078039248474e-05, "val/num_eos_tokens": 377, "lr": 4.90650566784549e-06, "episode": 10528, "epoch": 0.5317171717171717, "step": 658}, {"eps": 0, "objective/kl": 32.11946105957031, "objective/entropy": -46.258995056152344, "objective/non_score_reward": -1.6059731245040894, "objective/rlhf_reward": 12.776840209960938, "objective/scores": 14.375, "policy/approxkl_avg": 0.0012434273958206177, "policy/clipfrac_avg": 0.008764926344156265, "loss/policy_avg": 0.04267742112278938, "loss/value_avg": 0.2801401913166046, "val/clipfrac_avg": 0.07641617208719254, "policy/entropy_avg": 0.1816234588623047, "val/ratio": 0.9873616099357605, "val/ratio_var": 0.0001170080722658895, "val/num_eos_tokens": 16, "lr": 4.893151241063493e-06, "episode": 10544, "epoch": 0.5325252525252525, "step": 659}, {"eps": 0, "objective/kl": 39.09260559082031, "objective/entropy": -151.64088439941406, "objective/non_score_reward": -1.9546302556991577, "objective/rlhf_reward": 11.217245101928711, "objective/scores": 13.1875, "policy/approxkl_avg": 0.0007958562346175313, "policy/clipfrac_avg": 0.00742696039378643, "loss/policy_avg": 0.20670434832572937, "loss/value_avg": 0.484742134809494, "val/clipfrac_avg": 0.07210306078195572, "policy/entropy_avg": 0.1595144271850586, "val/ratio": 0.9886982440948486, "val/ratio_var": 4.408901440910995e-05, "val/num_eos_tokens": 20, "lr": 4.879797576806915e-06, "episode": 10560, "epoch": 0.5333333333333333, "step": 660}, {"eps": 0, "objective/kl": 39.04478073120117, "objective/entropy": -82.99937438964844, "objective/non_score_reward": -1.9522391557693481, "objective/rlhf_reward": 14.461823463439941, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008098644902929664, "policy/clipfrac_avg": 0.00760706327855587, "loss/policy_avg": -0.01065884530544281, "loss/value_avg": 0.23517704010009766, "val/clipfrac_avg": 0.13296082615852356, "policy/entropy_avg": 0.15206003189086914, "val/ratio": 0.9895342588424683, "val/ratio_var": 4.8955425882013515e-05, "val/num_eos_tokens": 369, "lr": 4.8664447703740905e-06, "episode": 10576, "epoch": 0.5341414141414141, "step": 661}, {"eps": 0, "objective/kl": 33.77119827270508, "objective/entropy": -50.7081413269043, "objective/non_score_reward": -1.6885600090026855, "objective/rlhf_reward": 13.561440467834473, "objective/scores": 15.25, "policy/approxkl_avg": 0.00106827134732157, "policy/clipfrac_avg": 0.005940725561231375, "loss/policy_avg": 0.2483931928873062, "loss/value_avg": 0.3836213946342468, "val/clipfrac_avg": 0.08841180801391602, "policy/entropy_avg": 0.18691349029541016, "val/ratio": 0.9869168400764465, "val/ratio_var": 0.00010277970432071015, "val/num_eos_tokens": 336, "lr": 4.853092917057224e-06, "episode": 10592, "epoch": 0.534949494949495, "step": 662}, {"eps": 0, "objective/kl": 40.23773193359375, "objective/entropy": -22.153806686401367, "objective/non_score_reward": -2.0118865966796875, "objective/rlhf_reward": 12.659988403320312, "objective/scores": 14.6875, "policy/approxkl_avg": 0.001191452844068408, "policy/clipfrac_avg": 0.0037520253099501133, "loss/policy_avg": 0.018621079623699188, "loss/value_avg": 0.47118768095970154, "val/clipfrac_avg": 0.13863703608512878, "policy/entropy_avg": 0.2298755645751953, "val/ratio": 0.984135627746582, "val/ratio_var": 8.8629407400731e-05, "val/num_eos_tokens": 143, "lr": 4.839742112141725e-06, "episode": 10608, "epoch": 0.5357575757575758, "step": 663}, {"eps": 0, "objective/kl": 32.980804443359375, "objective/entropy": -47.15129852294922, "objective/non_score_reward": -1.6490402221679688, "objective/rlhf_reward": 12.577522277832031, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009336364455521107, "policy/clipfrac_avg": 0.010629156604409218, "loss/policy_avg": -0.1973285973072052, "loss/value_avg": 0.8294652700424194, "val/clipfrac_avg": 0.04402673989534378, "policy/entropy_avg": 0.17932701110839844, "val/ratio": 0.988549530506134, "val/ratio_var": 7.59767135605216e-05, "val/num_eos_tokens": 230, "lr": 4.8263924509055185e-06, "episode": 10624, "epoch": 0.5365656565656566, "step": 664}, {"eps": 0, "objective/kl": 35.82035827636719, "objective/entropy": -92.09422302246094, "objective/non_score_reward": -1.7910178899765015, "objective/rlhf_reward": 15.990232467651367, "objective/scores": 17.75, "policy/approxkl_avg": 0.0008464535931125283, "policy/clipfrac_avg": 0.0052949972450733185, "loss/policy_avg": -0.3642651438713074, "loss/value_avg": 0.49642229080200195, "val/clipfrac_avg": 0.18585266172885895, "policy/entropy_avg": 0.16996145248413086, "val/ratio": 0.9879733920097351, "val/ratio_var": 7.15731002856046e-05, "val/num_eos_tokens": 760, "lr": 4.8130440286183725e-06, "episode": 10640, "epoch": 0.5373737373737374, "step": 665}, {"eps": 0, "objective/kl": 36.84406280517578, "objective/entropy": -100.74314880371094, "objective/non_score_reward": -1.842203140258789, "objective/rlhf_reward": 13.720296859741211, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0008232034742832184, "policy/clipfrac_avg": 0.005975116975605488, "loss/policy_avg": 0.19750815629959106, "loss/value_avg": 0.726028561592102, "val/clipfrac_avg": 0.0833563357591629, "policy/entropy_avg": 0.17266035079956055, "val/ratio": 0.9884688258171082, "val/ratio_var": 5.402052556746639e-05, "val/num_eos_tokens": 58, "lr": 4.799696940541204e-06, "episode": 10656, "epoch": 0.5381818181818182, "step": 666}, {"eps": 0, "objective/kl": 34.524009704589844, "objective/entropy": -124.33128356933594, "objective/non_score_reward": -1.7262005805969238, "objective/rlhf_reward": 11.640987396240234, "objective/scores": 13.375, "policy/approxkl_avg": 0.0007272096117958426, "policy/clipfrac_avg": 0.008608815260231495, "loss/policy_avg": -0.3925812244415283, "loss/value_avg": 0.9663629531860352, "val/clipfrac_avg": 0.03538993000984192, "policy/entropy_avg": 0.14444184303283691, "val/ratio": 0.9909723997116089, "val/ratio_var": 4.3006653868360445e-05, "val/num_eos_tokens": 439, "lr": 4.786351281925417e-06, "episode": 10672, "epoch": 0.538989898989899, "step": 667}, {"eps": 0, "objective/kl": 35.084346771240234, "objective/entropy": -34.56883239746094, "objective/non_score_reward": -1.7542171478271484, "objective/rlhf_reward": 15.206720352172852, "objective/scores": 17.0, "policy/approxkl_avg": 0.0011885869316756725, "policy/clipfrac_avg": 0.006960581988096237, "loss/policy_avg": -0.06382110714912415, "loss/value_avg": 0.3970024287700653, "val/clipfrac_avg": 0.13442066311836243, "policy/entropy_avg": 0.198333740234375, "val/ratio": 0.986121654510498, "val/ratio_var": 3.918609945685603e-05, "val/num_eos_tokens": 187, "lr": 4.773007148012207e-06, "episode": 10688, "epoch": 0.5397979797979798, "step": 668}, {"eps": 0, "objective/kl": 34.169227600097656, "objective/entropy": -87.38048553466797, "objective/non_score_reward": -1.7084612846374512, "objective/rlhf_reward": 12.05716323852539, "objective/scores": 13.75, "policy/approxkl_avg": 0.0009187318501062691, "policy/clipfrac_avg": 0.005587933585047722, "loss/policy_avg": 0.16306787729263306, "loss/value_avg": 0.45442262291908264, "val/clipfrac_avg": 0.19067704677581787, "policy/entropy_avg": 0.17048358917236328, "val/ratio": 0.9888883233070374, "val/ratio_var": 6.463148747570813e-05, "val/num_eos_tokens": 373, "lr": 4.759664634031897e-06, "episode": 10704, "epoch": 0.5406060606060606, "step": 669}, {"eps": 0, "objective/kl": 36.53352355957031, "objective/entropy": -84.40043640136719, "objective/non_score_reward": -1.8266760110855103, "objective/rlhf_reward": 13.681137084960938, "objective/scores": 15.5, "policy/approxkl_avg": 0.0009138257009908557, "policy/clipfrac_avg": 0.006062377244234085, "loss/policy_avg": 0.010326467454433441, "loss/value_avg": 0.3474205434322357, "val/clipfrac_avg": 0.09507843852043152, "policy/entropy_avg": 0.1702251434326172, "val/ratio": 0.9881380200386047, "val/ratio_var": 4.7510071453871205e-05, "val/num_eos_tokens": 295, "lr": 4.74632383520324e-06, "episode": 10720, "epoch": 0.5414141414141415, "step": 670}, {"eps": 0, "objective/kl": 39.167991638183594, "objective/entropy": -81.22120666503906, "objective/non_score_reward": -1.9583998918533325, "objective/rlhf_reward": 13.49472427368164, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0009018934797495604, "policy/clipfrac_avg": 0.015163691714406013, "loss/policy_avg": -0.09906017780303955, "loss/value_avg": 0.8066251873970032, "val/clipfrac_avg": 0.0449674017727375, "policy/entropy_avg": 0.17831039428710938, "val/ratio": 0.988625168800354, "val/ratio_var": 7.142542017390952e-05, "val/num_eos_tokens": 582, "lr": 4.732984846732755e-06, "episode": 10736, "epoch": 0.5422222222222223, "step": 671}, {"eps": 0, "objective/kl": 37.01219940185547, "objective/entropy": -144.14218139648438, "objective/non_score_reward": -1.8506097793579102, "objective/rlhf_reward": 14.227514266967773, "objective/scores": 16.125, "policy/approxkl_avg": 0.0008922894485294819, "policy/clipfrac_avg": 0.006483469158411026, "loss/policy_avg": -0.2206135392189026, "loss/value_avg": 0.23615601658821106, "val/clipfrac_avg": 0.07687856256961823, "policy/entropy_avg": 0.17594242095947266, "val/ratio": 0.9886537194252014, "val/ratio_var": 6.714969640597701e-05, "val/num_eos_tokens": 513, "lr": 4.719647763814041e-06, "episode": 10752, "epoch": 0.5430303030303031, "step": 672}, {"eps": 0, "objective/kl": 39.09211349487305, "objective/entropy": -88.67999267578125, "objective/non_score_reward": -1.9546055793762207, "objective/rlhf_reward": 13.686018943786621, "objective/scores": 15.625, "policy/approxkl_avg": 0.0010067424736917019, "policy/clipfrac_avg": 0.006908165290951729, "loss/policy_avg": -0.077633798122406, "loss/value_avg": 0.6793819665908813, "val/clipfrac_avg": 0.077333003282547, "policy/entropy_avg": 0.1997241973876953, "val/ratio": 0.9873216152191162, "val/ratio_var": 6.026164919603616e-05, "val/num_eos_tokens": 119, "lr": 4.7063126816270965e-06, "episode": 10768, "epoch": 0.5438383838383838, "step": 673}, {"eps": 0, "objective/kl": 30.964773178100586, "objective/entropy": -60.31924819946289, "objective/non_score_reward": -1.548238754272461, "objective/rlhf_reward": 12.936136245727539, "objective/scores": 14.5, "policy/approxkl_avg": 0.0009357638191431761, "policy/clipfrac_avg": 0.011794875375926495, "loss/policy_avg": 0.16710957884788513, "loss/value_avg": 0.7977907657623291, "val/clipfrac_avg": 0.143951416015625, "policy/entropy_avg": 0.17830944061279297, "val/ratio": 0.9884299635887146, "val/ratio_var": 9.555782162351534e-05, "val/num_eos_tokens": 346, "lr": 4.6929796953376395e-06, "episode": 10784, "epoch": 0.5446464646464646, "step": 674}, {"eps": 0, "objective/kl": 32.99176788330078, "objective/entropy": -66.3176498413086, "objective/non_score_reward": -1.6495885848999023, "objective/rlhf_reward": 14.381661415100098, "objective/scores": 16.0, "policy/approxkl_avg": 0.0007904217927716672, "policy/clipfrac_avg": 0.004523607436567545, "loss/policy_avg": -0.023821208626031876, "loss/value_avg": 0.22992631793022156, "val/clipfrac_avg": 0.05289379507303238, "policy/entropy_avg": 0.1616344451904297, "val/ratio": 0.9895982146263123, "val/ratio_var": 3.7571215216303244e-05, "val/num_eos_tokens": 387, "lr": 4.679648900096436e-06, "episode": 10800, "epoch": 0.5454545454545454, "step": 675}, {"eps": 0, "objective/kl": 34.93087387084961, "objective/entropy": -1.3176898956298828, "objective/non_score_reward": -1.7465436458587646, "objective/rlhf_reward": 14.097206115722656, "objective/scores": 15.875, "policy/approxkl_avg": 0.0011160490103065968, "policy/clipfrac_avg": 0.006563505623489618, "loss/policy_avg": 0.029874496161937714, "loss/value_avg": 0.32752853631973267, "val/clipfrac_avg": 0.10146623104810715, "policy/entropy_avg": 0.20054244995117188, "val/ratio": 0.9860273003578186, "val/ratio_var": 3.4691103792283684e-05, "val/num_eos_tokens": 224, "lr": 4.666320391038612e-06, "episode": 10816, "epoch": 0.5462626262626262, "step": 676}, {"eps": 0, "objective/kl": 36.158023834228516, "objective/entropy": -96.90899658203125, "objective/non_score_reward": -1.8079012632369995, "objective/rlhf_reward": 13.324911117553711, "objective/scores": 15.125, "policy/approxkl_avg": 0.0008718149038031697, "policy/clipfrac_avg": 0.009348929859697819, "loss/policy_avg": -0.10157568752765656, "loss/value_avg": 0.4907248616218567, "val/clipfrac_avg": 0.020003557205200195, "policy/entropy_avg": 0.15552330017089844, "val/ratio": 0.9896889925003052, "val/ratio_var": 5.074829823570326e-05, "val/num_eos_tokens": 119, "lr": 4.652994263282979e-06, "episode": 10832, "epoch": 0.547070707070707, "step": 677}, {"eps": 0, "objective/kl": 31.89101219177246, "objective/entropy": -82.16497802734375, "objective/non_score_reward": -1.5945507287979126, "objective/rlhf_reward": 15.303886413574219, "objective/scores": 16.875, "policy/approxkl_avg": 0.0011360663920640945, "policy/clipfrac_avg": 0.007264793384820223, "loss/policy_avg": -0.263140469789505, "loss/value_avg": 0.3839453458786011, "val/clipfrac_avg": 0.09376704692840576, "policy/entropy_avg": 0.1928253173828125, "val/ratio": 0.9877759218215942, "val/ratio_var": 5.5250733566936105e-05, "val/num_eos_tokens": 803, "lr": 4.6396706119313526e-06, "episode": 10848, "epoch": 0.5478787878787879, "step": 678}, {"eps": 0, "objective/kl": 37.956764221191406, "objective/entropy": -66.71647644042969, "objective/non_score_reward": -1.8978383541107178, "objective/rlhf_reward": 14.820911407470703, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009447145857848227, "policy/clipfrac_avg": 0.004722379148006439, "loss/policy_avg": -0.2529081106185913, "loss/value_avg": 1.246268391609192, "val/clipfrac_avg": 0.15111136436462402, "policy/entropy_avg": 0.16110455989837646, "val/ratio": 0.9897333979606628, "val/ratio_var": 5.177267303224653e-05, "val/num_eos_tokens": 376, "lr": 4.626349532067879e-06, "episode": 10864, "epoch": 0.5486868686868687, "step": 679}, {"eps": 0, "objective/kl": 35.02674102783203, "objective/entropy": -96.83143615722656, "objective/non_score_reward": -1.7513370513916016, "objective/rlhf_reward": 11.631475448608398, "objective/scores": 13.375, "policy/approxkl_avg": 0.0008016076171770692, "policy/clipfrac_avg": 0.008088190108537674, "loss/policy_avg": 0.2604403495788574, "loss/value_avg": 0.9619290232658386, "val/clipfrac_avg": 0.1873883306980133, "policy/entropy_avg": 0.14316558837890625, "val/ratio": 0.9903253316879272, "val/ratio_var": 3.635772009147331e-05, "val/num_eos_tokens": 55, "lr": 4.613031118758351e-06, "episode": 10880, "epoch": 0.5494949494949495, "step": 680}, {"eps": 0, "objective/kl": 35.87485885620117, "objective/entropy": -46.69654846191406, "objective/non_score_reward": -1.7937428951263428, "objective/rlhf_reward": 12.956256866455078, "objective/scores": 14.75, "policy/approxkl_avg": 0.0011159505229443312, "policy/clipfrac_avg": 0.007098098285496235, "loss/policy_avg": 0.20710928738117218, "loss/value_avg": 0.8212553262710571, "val/clipfrac_avg": 0.1026269719004631, "policy/entropy_avg": 0.18874073028564453, "val/ratio": 0.9870556592941284, "val/ratio_var": 4.734138565254398e-05, "val/num_eos_tokens": 32, "lr": 4.599715467049534e-06, "episode": 10896, "epoch": 0.5503030303030303, "step": 681}, {"eps": 0, "objective/kl": 45.23626708984375, "objective/entropy": -150.3234405517578, "objective/non_score_reward": -2.2618134021759033, "objective/rlhf_reward": 12.62881088256836, "objective/scores": 14.875, "policy/approxkl_avg": 0.0013120709918439388, "policy/clipfrac_avg": 0.0051262169145047665, "loss/policy_avg": -0.19795550405979156, "loss/value_avg": 0.36300960183143616, "val/clipfrac_avg": 0.09302499890327454, "policy/entropy_avg": 0.15036582946777344, "val/ratio": 0.9899913668632507, "val/ratio_var": 5.3359748562797904e-05, "val/num_eos_tokens": 451, "lr": 4.5864026719684796e-06, "episode": 10912, "epoch": 0.5511111111111111, "step": 682}, {"eps": 0, "objective/kl": 31.69464111328125, "objective/entropy": -142.62588500976562, "objective/non_score_reward": -1.584732174873352, "objective/rlhf_reward": 14.212142944335938, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0007317900890484452, "policy/clipfrac_avg": 0.01235207635909319, "loss/policy_avg": 0.1822894662618637, "loss/value_avg": 0.8603230118751526, "val/clipfrac_avg": 0.14113879203796387, "policy/entropy_avg": 0.13785886764526367, "val/ratio": 0.9909247159957886, "val/ratio_var": 5.3253872465575114e-05, "val/num_eos_tokens": 289, "lr": 4.573092828521857e-06, "episode": 10928, "epoch": 0.5519191919191919, "step": 683}, {"eps": 0, "objective/kl": 43.83638000488281, "objective/entropy": -27.279052734375, "objective/non_score_reward": -2.191819190979004, "objective/rlhf_reward": 14.433180809020996, "objective/scores": 16.625, "policy/approxkl_avg": 0.0010979644721373916, "policy/clipfrac_avg": 0.005808757618069649, "loss/policy_avg": -0.0519242025911808, "loss/value_avg": 0.19368618726730347, "val/clipfrac_avg": 0.00993678905069828, "policy/entropy_avg": 0.21721267700195312, "val/ratio": 0.9861806631088257, "val/ratio_var": 3.975445360993035e-05, "val/num_eos_tokens": 24, "lr": 4.559786031695275e-06, "episode": 10944, "epoch": 0.5527272727272727, "step": 684}, {"eps": 0, "objective/kl": 26.137264251708984, "objective/entropy": -114.476806640625, "objective/non_score_reward": -1.3068633079528809, "objective/rlhf_reward": 12.943137168884277, "objective/scores": 14.25, "policy/approxkl_avg": 0.0007893022848293185, "policy/clipfrac_avg": 0.0076246787793934345, "loss/policy_avg": 0.11247897148132324, "loss/value_avg": 0.3687130808830261, "val/clipfrac_avg": 0.0864788144826889, "policy/entropy_avg": 0.14629364013671875, "val/ratio": 0.9891906976699829, "val/ratio_var": 7.146107236621901e-05, "val/num_eos_tokens": 446, "lr": 4.546482376452589e-06, "episode": 10960, "epoch": 0.5535353535353535, "step": 685}, {"eps": 0, "objective/kl": 28.205734252929688, "objective/entropy": -129.17437744140625, "objective/non_score_reward": -1.4102866649627686, "objective/rlhf_reward": 11.988150596618652, "objective/scores": 13.375, "policy/approxkl_avg": 0.0007981459493748844, "policy/clipfrac_avg": 0.004403797909617424, "loss/policy_avg": -0.035443827509880066, "loss/value_avg": 0.3387452960014343, "val/clipfrac_avg": 0.05607829615473747, "policy/entropy_avg": 0.15001487731933594, "val/ratio": 0.9896979331970215, "val/ratio_var": 4.439734038896859e-05, "val/num_eos_tokens": 331, "lr": 4.533181957735247e-06, "episode": 10976, "epoch": 0.5543434343434344, "step": 686}, {"eps": 0, "objective/kl": 34.03411865234375, "objective/entropy": -69.12667083740234, "objective/non_score_reward": -1.7017061710357666, "objective/rlhf_reward": 13.399856567382812, "objective/scores": 15.125, "policy/approxkl_avg": 0.0009728028671815991, "policy/clipfrac_avg": 0.005738085135817528, "loss/policy_avg": 0.03433602303266525, "loss/value_avg": 0.7930219173431396, "val/clipfrac_avg": 0.11831800639629364, "policy/entropy_avg": 0.19457244873046875, "val/ratio": 0.9865483045578003, "val/ratio_var": 7.133706822060049e-05, "val/num_eos_tokens": 44, "lr": 4.5198848704615915e-06, "episode": 10992, "epoch": 0.5551515151515152, "step": 687}, {"eps": 0, "objective/kl": 30.15737533569336, "objective/entropy": -71.97180938720703, "objective/non_score_reward": -1.507868766784668, "objective/rlhf_reward": 13.203069686889648, "objective/scores": 14.6875, "policy/approxkl_avg": 0.0008744328515604138, "policy/clipfrac_avg": 0.005093730054795742, "loss/policy_avg": -0.22896869480609894, "loss/value_avg": 0.9052561521530151, "val/clipfrac_avg": 0.08035694062709808, "policy/entropy_avg": 0.15955305099487305, "val/ratio": 0.9889332056045532, "val/ratio_var": 3.423935777391307e-05, "val/num_eos_tokens": 290, "lr": 4.506591209526195e-06, "episode": 11008, "epoch": 0.555959595959596, "step": 688}, {"eps": 0, "objective/kl": 33.093013763427734, "objective/entropy": -78.7776107788086, "objective/non_score_reward": -1.6546508073806763, "objective/rlhf_reward": 14.282849311828613, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0009888019412755966, "policy/clipfrac_avg": 0.00978151336312294, "loss/policy_avg": 0.03792828321456909, "loss/value_avg": 0.9123845100402832, "val/clipfrac_avg": 0.07123228162527084, "policy/entropy_avg": 0.17550277709960938, "val/ratio": 0.9879224300384521, "val/ratio_var": 6.044195106369443e-05, "val/num_eos_tokens": 16, "lr": 4.493301069799171e-06, "episode": 11024, "epoch": 0.5567676767676768, "step": 689}, {"eps": 0, "objective/kl": 33.34143829345703, "objective/entropy": 5.367833614349365, "objective/non_score_reward": -1.667072057723999, "objective/rlhf_reward": 13.621990203857422, "objective/scores": 15.3125, "policy/approxkl_avg": 0.001577872084453702, "policy/clipfrac_avg": 0.009263614192605019, "loss/policy_avg": 0.015539663843810558, "loss/value_avg": 0.3836192786693573, "val/clipfrac_avg": 0.05502686649560928, "policy/entropy_avg": 0.2828826904296875, "val/ratio": 0.9818495512008667, "val/ratio_var": 7.278368866536766e-05, "val/num_eos_tokens": 182, "lr": 4.480014546125511e-06, "episode": 11040, "epoch": 0.5575757575757576, "step": 690}, {"eps": 0, "objective/kl": 33.737091064453125, "objective/entropy": -61.70869064331055, "objective/non_score_reward": -1.686854600906372, "objective/rlhf_reward": 14.25845718383789, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0008769157575443387, "policy/clipfrac_avg": 0.0052274661138653755, "loss/policy_avg": -0.0511251725256443, "loss/value_avg": 0.38562020659446716, "val/clipfrac_avg": 0.032033294439315796, "policy/entropy_avg": 0.17685508728027344, "val/ratio": 0.9887614250183105, "val/ratio_var": 4.9923761253012344e-05, "val/num_eos_tokens": 406, "lr": 4.466731733324399e-06, "episode": 11056, "epoch": 0.5583838383838384, "step": 691}, {"eps": 0, "objective/kl": 31.425121307373047, "objective/entropy": -69.50019836425781, "objective/non_score_reward": -1.571256160736084, "objective/rlhf_reward": 12.920930862426758, "objective/scores": 14.5, "policy/approxkl_avg": 0.000965589948464185, "policy/clipfrac_avg": 0.009000038728117943, "loss/policy_avg": 0.29204708337783813, "loss/value_avg": 0.5752546787261963, "val/clipfrac_avg": 0.09444969892501831, "policy/entropy_avg": 0.18520164489746094, "val/ratio": 0.9867506623268127, "val/ratio_var": 9.781523112906143e-05, "val/num_eos_tokens": 42, "lr": 4.4534527261885365e-06, "episode": 11072, "epoch": 0.5591919191919192, "step": 692}, {"eps": 0, "objective/kl": 31.978641510009766, "objective/entropy": -95.70045471191406, "objective/non_score_reward": -1.5989320278167725, "objective/rlhf_reward": 13.791692733764648, "objective/scores": 15.375, "policy/approxkl_avg": 0.0008498022216372192, "policy/clipfrac_avg": 0.014874793589115143, "loss/policy_avg": -0.304983526468277, "loss/value_avg": 0.4396636188030243, "val/clipfrac_avg": 0.08384636044502258, "policy/entropy_avg": 0.15596485137939453, "val/ratio": 0.989993691444397, "val/ratio_var": 7.268232002388686e-05, "val/num_eos_tokens": 116, "lr": 4.4401776194834615e-06, "episode": 11088, "epoch": 0.56, "step": 693}, {"eps": 0, "objective/kl": 29.767484664916992, "objective/entropy": -68.41473388671875, "objective/non_score_reward": -1.4883742332458496, "objective/rlhf_reward": 12.292876243591309, "objective/scores": 13.75, "policy/approxkl_avg": 0.0010167569853365421, "policy/clipfrac_avg": 0.01529286801815033, "loss/policy_avg": -0.28100529313087463, "loss/value_avg": 0.36995184421539307, "val/clipfrac_avg": 0.14090892672538757, "policy/entropy_avg": 0.18943500518798828, "val/ratio": 0.9882405400276184, "val/ratio_var": 9.475964907323942e-05, "val/num_eos_tokens": 18, "lr": 4.426906507946881e-06, "episode": 11104, "epoch": 0.5608080808080808, "step": 694}, {"eps": 0, "objective/kl": 32.63812255859375, "objective/entropy": -12.017631530761719, "objective/non_score_reward": -1.6319061517715454, "objective/rlhf_reward": 15.243093490600586, "objective/scores": 16.875, "policy/approxkl_avg": 0.0010099872015416622, "policy/clipfrac_avg": 0.004084677901118994, "loss/policy_avg": -0.06622211635112762, "loss/value_avg": 0.334886372089386, "val/clipfrac_avg": 0.062321148812770844, "policy/entropy_avg": 0.18613624572753906, "val/ratio": 0.987127423286438, "val/ratio_var": 4.299115244066343e-05, "val/num_eos_tokens": 267, "lr": 4.413639486287992e-06, "episode": 11120, "epoch": 0.5616161616161616, "step": 695}, {"eps": 0, "objective/kl": 28.685632705688477, "objective/entropy": -87.83956909179688, "objective/non_score_reward": -1.4342817068099976, "objective/rlhf_reward": 11.417280197143555, "objective/scores": 12.875, "policy/approxkl_avg": 0.0008759163902141154, "policy/clipfrac_avg": 0.006767658516764641, "loss/policy_avg": -0.06522263586521149, "loss/value_avg": 0.7103580236434937, "val/clipfrac_avg": 0.18274252116680145, "policy/entropy_avg": 0.19121313095092773, "val/ratio": 0.9897185564041138, "val/ratio_var": 5.892527042306028e-05, "val/num_eos_tokens": 573, "lr": 4.4003766491867984e-06, "episode": 11136, "epoch": 0.5624242424242424, "step": 696}, {"eps": 0, "objective/kl": 33.15848159790039, "objective/entropy": -99.48849487304688, "objective/non_score_reward": -1.6579241752624512, "objective/rlhf_reward": 11.59207534790039, "objective/scores": 13.25, "policy/approxkl_avg": 0.0008881977992132306, "policy/clipfrac_avg": 0.007622114382684231, "loss/policy_avg": 0.1779220849275589, "loss/value_avg": 0.6948210000991821, "val/clipfrac_avg": 0.13009925186634064, "policy/entropy_avg": 0.16399192810058594, "val/ratio": 0.9894096255302429, "val/ratio_var": 4.7340425226138905e-05, "val/num_eos_tokens": 59, "lr": 4.3871180912934465e-06, "episode": 11152, "epoch": 0.5632323232323232, "step": 697}, {"eps": 0, "objective/kl": 28.16010093688965, "objective/entropy": -121.6069564819336, "objective/non_score_reward": -1.4080051183700562, "objective/rlhf_reward": 14.021682739257812, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0007813716074451804, "policy/clipfrac_avg": 0.010615997016429901, "loss/policy_avg": 0.22691625356674194, "loss/value_avg": 0.518365740776062, "val/clipfrac_avg": 0.04978542774915695, "policy/entropy_avg": 0.1465442180633545, "val/ratio": 0.9906186461448669, "val/ratio_var": 7.280353020178154e-05, "val/num_eos_tokens": 567, "lr": 4.373863907227539e-06, "episode": 11168, "epoch": 0.564040404040404, "step": 698}, {"eps": 0, "objective/kl": 29.718286514282227, "objective/entropy": -56.37757110595703, "objective/non_score_reward": -1.4859144687652588, "objective/rlhf_reward": 14.08439826965332, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0009621034259907901, "policy/clipfrac_avg": 0.011594917625188828, "loss/policy_avg": 0.4753793179988861, "loss/value_avg": 0.8896934390068054, "val/clipfrac_avg": 0.028925057500600815, "policy/entropy_avg": 0.15114402770996094, "val/ratio": 0.9886069297790527, "val/ratio_var": 5.0333743274677545e-05, "val/num_eos_tokens": 93, "lr": 4.3606141915774695e-06, "episode": 11184, "epoch": 0.5648484848484848, "step": 699}, {"eps": 0, "objective/kl": 30.32842445373535, "objective/entropy": -51.717899322509766, "objective/non_score_reward": -1.5164213180541992, "objective/rlhf_reward": 11.1085786819458, "objective/scores": 12.625, "policy/approxkl_avg": 0.0010908711701631546, "policy/clipfrac_avg": 0.005067873280495405, "loss/policy_avg": 0.013627409934997559, "loss/value_avg": 0.39067256450653076, "val/clipfrac_avg": 0.05506103113293648, "policy/entropy_avg": 0.21241378784179688, "val/ratio": 0.9856171607971191, "val/ratio_var": 8.688835805514827e-05, "val/num_eos_tokens": 84, "lr": 4.347369038899744e-06, "episode": 11200, "epoch": 0.5656565656565656, "step": 700}, {"eps": 0, "objective/kl": 33.42132568359375, "objective/entropy": -74.95745849609375, "objective/non_score_reward": -1.6710662841796875, "objective/rlhf_reward": 14.828933715820312, "objective/scores": 16.5, "policy/approxkl_avg": 0.0013184514828026295, "policy/clipfrac_avg": 0.00809475127607584, "loss/policy_avg": -0.09055965393781662, "loss/value_avg": 0.5455071926116943, "val/clipfrac_avg": 0.18833743035793304, "policy/entropy_avg": 0.17902374267578125, "val/ratio": 0.9879262447357178, "val/ratio_var": 6.120221951277927e-05, "val/num_eos_tokens": 358, "lr": 4.334128543718295e-06, "episode": 11216, "epoch": 0.5664646464646464, "step": 701}, {"eps": 0, "objective/kl": 27.537485122680664, "objective/entropy": -90.61648559570312, "objective/non_score_reward": -1.3768744468688965, "objective/rlhf_reward": 12.826250076293945, "objective/scores": 14.1875, "policy/approxkl_avg": 0.0009536159923300147, "policy/clipfrac_avg": 0.01545755285769701, "loss/policy_avg": 0.2261451780796051, "loss/value_avg": 1.5403046607971191, "val/clipfrac_avg": 0.033450622111558914, "policy/entropy_avg": 0.20562219619750977, "val/ratio": 0.9873440265655518, "val/ratio_var": 7.613677007611841e-05, "val/num_eos_tokens": 595, "lr": 4.320892800523827e-06, "episode": 11232, "epoch": 0.5672727272727273, "step": 702}, {"eps": 0, "objective/kl": 43.216548919677734, "objective/entropy": -56.119659423828125, "objective/non_score_reward": -2.160827398300171, "objective/rlhf_reward": 15.80792236328125, "objective/scores": 18.0, "policy/approxkl_avg": 0.0010682737920433283, "policy/clipfrac_avg": 0.005114596337080002, "loss/policy_avg": -0.05153674632310867, "loss/value_avg": 0.18954747915267944, "val/clipfrac_avg": 0.12178023159503937, "policy/entropy_avg": 0.20781707763671875, "val/ratio": 0.9860707521438599, "val/ratio_var": 5.392924140323885e-05, "val/num_eos_tokens": 244, "lr": 4.307661903773129e-06, "episode": 11248, "epoch": 0.5680808080808081, "step": 703}, {"eps": 0, "objective/kl": 32.27890396118164, "objective/entropy": -180.00924682617188, "objective/non_score_reward": -1.6139451265335083, "objective/rlhf_reward": 12.425117492675781, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0007389815291389823, "policy/clipfrac_avg": 0.00613048393279314, "loss/policy_avg": 0.22105136513710022, "loss/value_avg": 0.43452760577201843, "val/clipfrac_avg": 0.12273523956537247, "policy/entropy_avg": 0.15887287259101868, "val/ratio": 0.9900252819061279, "val/ratio_var": 6.185931124491617e-05, "val/num_eos_tokens": 611, "lr": 4.294435947888405e-06, "episode": 11264, "epoch": 0.5688888888888889, "step": 704}, {"eps": 0, "objective/kl": 29.207317352294922, "objective/entropy": -73.2503890991211, "objective/non_score_reward": -1.460365891456604, "objective/rlhf_reward": 15.766195297241211, "objective/scores": 17.25, "policy/approxkl_avg": 0.0010228066239506006, "policy/clipfrac_avg": 0.010731351561844349, "loss/policy_avg": -0.044020622968673706, "loss/value_avg": 1.459969401359558, "val/clipfrac_avg": 0.1418685019016266, "policy/entropy_avg": 0.17015647888183594, "val/ratio": 0.9865360856056213, "val/ratio_var": 0.0001155980207840912, "val/num_eos_tokens": 378, "lr": 4.281215027256592e-06, "episode": 11280, "epoch": 0.5696969696969697, "step": 705}, {"eps": 0, "objective/kl": 39.329261779785156, "objective/entropy": -91.05632019042969, "objective/non_score_reward": -1.966463327407837, "objective/rlhf_reward": 12.744474411010742, "objective/scores": 14.6875, "policy/approxkl_avg": 0.0010600072564557195, "policy/clipfrac_avg": 0.005786027293652296, "loss/policy_avg": 0.10734500735998154, "loss/value_avg": 0.40052735805511475, "val/clipfrac_avg": 0.032500892877578735, "policy/entropy_avg": 0.23768234252929688, "val/ratio": 0.9860942959785461, "val/ratio_var": 5.682241317117587e-05, "val/num_eos_tokens": 203, "lr": 4.2679992362287e-06, "episode": 11296, "epoch": 0.5705050505050505, "step": 706}, {"eps": 0, "objective/kl": 34.540130615234375, "objective/entropy": -93.2535400390625, "objective/non_score_reward": -1.727006435394287, "objective/rlhf_reward": 14.788618087768555, "objective/scores": 16.5, "policy/approxkl_avg": 0.0008475214708596468, "policy/clipfrac_avg": 0.027025574818253517, "loss/policy_avg": -0.5062837600708008, "loss/value_avg": 0.7742470502853394, "val/clipfrac_avg": 0.1775030940771103, "policy/entropy_avg": 0.21597671508789062, "val/ratio": 0.9894334077835083, "val/ratio_var": 9.222733206115663e-05, "val/num_eos_tokens": 165, "lr": 4.254788669119127e-06, "episode": 11312, "epoch": 0.5713131313131313, "step": 707}, {"eps": 0, "objective/kl": 36.62909698486328, "objective/entropy": -110.37503051757812, "objective/non_score_reward": -1.831454873085022, "objective/rlhf_reward": 12.89510726928711, "objective/scores": 14.75, "policy/approxkl_avg": 0.0009251610608771443, "policy/clipfrac_avg": 0.00941617414355278, "loss/policy_avg": -0.1463669091463089, "loss/value_avg": 0.5632686614990234, "val/clipfrac_avg": 0.04514690116047859, "policy/entropy_avg": 0.16801166534423828, "val/ratio": 0.9897239208221436, "val/ratio_var": 7.692311191931367e-05, "val/num_eos_tokens": 267, "lr": 4.241583420204998e-06, "episode": 11328, "epoch": 0.5721212121212121, "step": 708}, {"eps": 0, "objective/kl": 31.900005340576172, "objective/entropy": -76.27204132080078, "objective/non_score_reward": -1.5950002670288086, "objective/rlhf_reward": 14.053436279296875, "objective/scores": 15.625, "policy/approxkl_avg": 0.0010020419722422957, "policy/clipfrac_avg": 0.011507621966302395, "loss/policy_avg": -0.35295569896698, "loss/value_avg": 0.7035583257675171, "val/clipfrac_avg": 0.12508229911327362, "policy/entropy_avg": 0.16324424743652344, "val/ratio": 0.9896982908248901, "val/ratio_var": 4.9723090342013165e-05, "val/num_eos_tokens": 173, "lr": 4.228383583725471e-06, "episode": 11344, "epoch": 0.572929292929293, "step": 709}, {"eps": 0, "objective/kl": 36.81852722167969, "objective/entropy": -79.55085754394531, "objective/non_score_reward": -1.8409266471862793, "objective/rlhf_reward": 12.159073829650879, "objective/scores": 14.0, "policy/approxkl_avg": 0.0011080782860517502, "policy/clipfrac_avg": 0.006503967102617025, "loss/policy_avg": 0.160404771566391, "loss/value_avg": 1.1914963722229004, "val/clipfrac_avg": 0.08140315115451813, "policy/entropy_avg": 0.19612884521484375, "val/ratio": 0.9866147041320801, "val/ratio_var": 7.869482942624018e-05, "val/num_eos_tokens": 538, "lr": 4.215189253881093e-06, "episode": 11360, "epoch": 0.5737373737373738, "step": 710}, {"eps": 0, "objective/kl": 34.31198501586914, "objective/entropy": -119.03164672851562, "objective/non_score_reward": -1.7155992984771729, "objective/rlhf_reward": 13.503150939941406, "objective/scores": 15.25, "policy/approxkl_avg": 0.0007362160831689835, "policy/clipfrac_avg": 0.00589254405349493, "loss/policy_avg": 0.07548795640468597, "loss/value_avg": 0.5138013362884521, "val/clipfrac_avg": 0.02668866142630577, "policy/entropy_avg": 0.14674091339111328, "val/ratio": 0.990146279335022, "val/ratio_var": 3.9673428545938805e-05, "val/num_eos_tokens": 515, "lr": 4.2020005248331056e-06, "episode": 11376, "epoch": 0.5745454545454546, "step": 711}, {"eps": 0, "objective/kl": 38.006500244140625, "objective/entropy": -112.06816101074219, "objective/non_score_reward": -1.900325059890747, "objective/rlhf_reward": 13.990300178527832, "objective/scores": 15.875, "policy/approxkl_avg": 0.000874843040946871, "policy/clipfrac_avg": 0.007544378750026226, "loss/policy_avg": -0.2925986051559448, "loss/value_avg": 0.3821415603160858, "val/clipfrac_avg": 0.025153353810310364, "policy/entropy_avg": 0.16875195503234863, "val/ratio": 0.9893897771835327, "val/ratio_var": 8.519255788996816e-05, "val/num_eos_tokens": 4, "lr": 4.1888174907027775e-06, "episode": 11392, "epoch": 0.5753535353535354, "step": 712}, {"eps": 0, "objective/kl": 35.69511413574219, "objective/entropy": -92.28114318847656, "objective/non_score_reward": -1.7847557067871094, "objective/rlhf_reward": 13.73868179321289, "objective/scores": 15.5, "policy/approxkl_avg": 0.0008441185345873237, "policy/clipfrac_avg": 0.005670716054737568, "loss/policy_avg": -0.5444605350494385, "loss/value_avg": 1.517474889755249, "val/clipfrac_avg": 0.1791025698184967, "policy/entropy_avg": 0.1591033935546875, "val/ratio": 0.9903852939605713, "val/ratio_var": 4.043114677187987e-05, "val/num_eos_tokens": 98, "lr": 4.175640245570742e-06, "episode": 11408, "epoch": 0.5761616161616162, "step": 713}, {"eps": 0, "objective/kl": 42.17440414428711, "objective/entropy": -193.77468872070312, "objective/non_score_reward": -2.108720302581787, "objective/rlhf_reward": 12.281905174255371, "objective/scores": 14.375, "policy/approxkl_avg": 0.0008298080647364259, "policy/clipfrac_avg": 0.00635810662060976, "loss/policy_avg": -0.3059786260128021, "loss/value_avg": 0.6795393228530884, "val/clipfrac_avg": 0.0064142923802137375, "policy/entropy_avg": 0.15823698043823242, "val/ratio": 0.9892975091934204, "val/ratio_var": 6.228621350601315e-05, "val/num_eos_tokens": 1666, "lr": 4.162468883476319e-06, "episode": 11424, "epoch": 0.576969696969697, "step": 714}, {"eps": 0, "objective/kl": 44.308345794677734, "objective/entropy": -22.63348388671875, "objective/non_score_reward": -2.2154173851013184, "objective/rlhf_reward": 14.82364559173584, "objective/scores": 17.0, "policy/approxkl_avg": 0.0012546201469376683, "policy/clipfrac_avg": 0.007735411170870066, "loss/policy_avg": -0.023257479071617126, "loss/value_avg": 0.18199321627616882, "val/clipfrac_avg": 0.0028473599813878536, "policy/entropy_avg": 0.240936279296875, "val/ratio": 0.9840776920318604, "val/ratio_var": 5.949061960563995e-05, "val/num_eos_tokens": 302, "lr": 4.149303498416838e-06, "episode": 11440, "epoch": 0.5777777777777777, "step": 715}, {"eps": 0, "objective/kl": 40.63113784790039, "objective/entropy": -168.4746856689453, "objective/non_score_reward": -2.031557083129883, "objective/rlhf_reward": 13.070005416870117, "objective/scores": 15.125, "policy/approxkl_avg": 0.0007934231543913484, "policy/clipfrac_avg": 0.012050900608301163, "loss/policy_avg": 0.3237575888633728, "loss/value_avg": 0.38197559118270874, "val/clipfrac_avg": 0.09322839975357056, "policy/entropy_avg": 0.14413070678710938, "val/ratio": 0.9900047779083252, "val/ratio_var": 5.29003118572291e-05, "val/num_eos_tokens": 1174, "lr": 4.136144184346979e-06, "episode": 11456, "epoch": 0.5785858585858585, "step": 716}, {"eps": 0, "objective/kl": 42.17526626586914, "objective/entropy": -96.77055358886719, "objective/non_score_reward": -2.1087632179260254, "objective/rlhf_reward": 14.500611305236816, "objective/scores": 16.625, "policy/approxkl_avg": 0.000815618084743619, "policy/clipfrac_avg": 0.008080050349235535, "loss/policy_avg": 0.23654325306415558, "loss/value_avg": 1.166879653930664, "val/clipfrac_avg": 0.02973807230591774, "policy/entropy_avg": 0.14806270599365234, "val/ratio": 0.9899397492408752, "val/ratio_var": 3.871042645187117e-05, "val/num_eos_tokens": 297, "lr": 4.122991035178093e-06, "episode": 11472, "epoch": 0.5793939393939394, "step": 717}, {"eps": 0, "objective/kl": 42.6419792175293, "objective/entropy": -58.205047607421875, "objective/non_score_reward": -2.132099151611328, "objective/rlhf_reward": 12.203838348388672, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0009566791704855859, "policy/clipfrac_avg": 0.0025268401950597763, "loss/policy_avg": -0.11968575417995453, "loss/value_avg": 0.9426589608192444, "val/clipfrac_avg": 0.1098824143409729, "policy/entropy_avg": 0.18893003463745117, "val/ratio": 0.9869214296340942, "val/ratio_var": 8.384165266761556e-05, "val/num_eos_tokens": 16, "lr": 4.109844144777537e-06, "episode": 11488, "epoch": 0.5802020202020202, "step": 718}, {"eps": 0, "objective/kl": 40.70527648925781, "objective/entropy": -105.71292114257812, "objective/non_score_reward": -2.035263776779175, "objective/rlhf_reward": 13.605360984802246, "objective/scores": 15.625, "policy/approxkl_avg": 0.0008132618968375027, "policy/clipfrac_avg": 0.004117591306567192, "loss/policy_avg": 0.0208362378180027, "loss/value_avg": 0.8499202728271484, "val/clipfrac_avg": 0.04368842393159866, "policy/entropy_avg": 0.15980243682861328, "val/ratio": 0.9895676374435425, "val/ratio_var": 6.342094275169075e-05, "val/num_eos_tokens": 428, "lr": 4.096703606968007e-06, "episode": 11504, "epoch": 0.581010101010101, "step": 719}, {"eps": 0, "objective/kl": 32.93721008300781, "objective/entropy": -100.23733520507812, "objective/non_score_reward": -1.6468604803085327, "objective/rlhf_reward": 15.134389877319336, "objective/scores": 16.75, "policy/approxkl_avg": 0.0008385346736758947, "policy/clipfrac_avg": 0.009182492271065712, "loss/policy_avg": -0.11627452820539474, "loss/value_avg": 0.7368168830871582, "val/clipfrac_avg": 0.06867726147174835, "policy/entropy_avg": 0.17890548706054688, "val/ratio": 0.9896779656410217, "val/ratio_var": 7.192800694610924e-05, "val/num_eos_tokens": 144, "lr": 4.083569515526853e-06, "episode": 11520, "epoch": 0.5818181818181818, "step": 720}, {"eps": 0, "objective/kl": 39.985321044921875, "objective/entropy": -103.3985366821289, "objective/non_score_reward": -1.9992659091949463, "objective/rlhf_reward": 14.547609329223633, "objective/scores": 16.5, "policy/approxkl_avg": 0.000881620217114687, "policy/clipfrac_avg": 0.004334163852035999, "loss/policy_avg": -0.06581735610961914, "loss/value_avg": 0.7933100461959839, "val/clipfrac_avg": 0.04114583134651184, "policy/entropy_avg": 0.1855926513671875, "val/ratio": 0.989449679851532, "val/ratio_var": 6.215622852323577e-05, "val/num_eos_tokens": 161, "lr": 4.070441964185428e-06, "episode": 11536, "epoch": 0.5826262626262626, "step": 721}, {"eps": 0, "objective/kl": 36.42585754394531, "objective/entropy": -119.77040100097656, "objective/non_score_reward": -1.8212929964065552, "objective/rlhf_reward": 13.467769622802734, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0008070225012488663, "policy/clipfrac_avg": 0.007022568956017494, "loss/policy_avg": 0.44041353464126587, "loss/value_avg": 0.7198789119720459, "val/clipfrac_avg": 0.05163312703371048, "policy/entropy_avg": 0.15318524837493896, "val/ratio": 0.9899815320968628, "val/ratio_var": 4.889031333732419e-05, "val/num_eos_tokens": 304, "lr": 4.05732104662841e-06, "episode": 11552, "epoch": 0.5834343434343434, "step": 722}, {"eps": 0, "objective/kl": 39.113037109375, "objective/entropy": -50.46599578857422, "objective/non_score_reward": -1.955651879310608, "objective/rlhf_reward": 13.911535263061523, "objective/scores": 15.875, "policy/approxkl_avg": 0.0010366745991632342, "policy/clipfrac_avg": 0.006084335036575794, "loss/policy_avg": 0.4866091012954712, "loss/value_avg": 1.1953067779541016, "val/clipfrac_avg": 0.09489050507545471, "policy/entropy_avg": 0.20149445533752441, "val/ratio": 0.9867548942565918, "val/ratio_var": 6.938407750567421e-05, "val/num_eos_tokens": 36, "lr": 4.04420685649314e-06, "episode": 11568, "epoch": 0.5842424242424242, "step": 723}, {"eps": 0, "objective/kl": 37.355255126953125, "objective/entropy": -112.03081512451172, "objective/non_score_reward": -1.8677629232406616, "objective/rlhf_reward": 13.49161148071289, "objective/scores": 15.375, "policy/approxkl_avg": 0.001035548048093915, "policy/clipfrac_avg": 0.006865138188004494, "loss/policy_avg": 0.1589544266462326, "loss/value_avg": 0.568170964717865, "val/clipfrac_avg": 0.05233292281627655, "policy/entropy_avg": 0.20564031600952148, "val/ratio": 0.9859846830368042, "val/ratio_var": 0.00010150864545721561, "val/num_eos_tokens": 347, "lr": 4.031099487368937e-06, "episode": 11584, "epoch": 0.585050505050505, "step": 724}, {"eps": 0, "objective/kl": 41.90481948852539, "objective/entropy": -69.79931640625, "objective/non_score_reward": -2.095241069793701, "objective/rlhf_reward": 14.85007095336914, "objective/scores": 17.0, "policy/approxkl_avg": 0.0010360007872805, "policy/clipfrac_avg": 0.004560987465083599, "loss/policy_avg": -0.19635428488254547, "loss/value_avg": 0.33023789525032043, "val/clipfrac_avg": 0.08375490456819534, "policy/entropy_avg": 0.20290184020996094, "val/ratio": 0.98649001121521, "val/ratio_var": 6.110723916208372e-05, "val/num_eos_tokens": 261, "lr": 4.0179990327964526e-06, "episode": 11600, "epoch": 0.5858585858585859, "step": 725}, {"eps": 0, "objective/kl": 32.01334762573242, "objective/entropy": -104.75285339355469, "objective/non_score_reward": -1.6006674766540527, "objective/rlhf_reward": 12.766520500183105, "objective/scores": 14.375, "policy/approxkl_avg": 0.0010034965816885233, "policy/clipfrac_avg": 0.007322699762880802, "loss/policy_avg": 0.0999169647693634, "loss/value_avg": 1.0625457763671875, "val/clipfrac_avg": 0.14562752842903137, "policy/entropy_avg": 0.19988441467285156, "val/ratio": 0.9887305498123169, "val/ratio_var": 7.558859942946583e-05, "val/num_eos_tokens": 33, "lr": 4.004905586266988e-06, "episode": 11616, "epoch": 0.5866666666666667, "step": 726}, {"eps": 0, "objective/kl": 43.23419952392578, "objective/entropy": -53.65510940551758, "objective/non_score_reward": -2.161710262298584, "objective/rlhf_reward": 12.986727714538574, "objective/scores": 15.125, "policy/approxkl_avg": 0.001010584644973278, "policy/clipfrac_avg": 0.006680442951619625, "loss/policy_avg": 0.004431502893567085, "loss/value_avg": 0.5651569366455078, "val/clipfrac_avg": 0.028638603165745735, "policy/entropy_avg": 0.17978286743164062, "val/ratio": 0.9880765676498413, "val/ratio_var": 3.508182999212295e-05, "val/num_eos_tokens": 25, "lr": 3.991819241221836e-06, "episode": 11632, "epoch": 0.5874747474747475, "step": 727}, {"eps": 0, "objective/kl": 40.000404357910156, "objective/entropy": -147.60580444335938, "objective/non_score_reward": -2.0000202655792236, "objective/rlhf_reward": 13.304666519165039, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0008220034651458263, "policy/clipfrac_avg": 0.010559421963989735, "loss/policy_avg": -0.026804931461811066, "loss/value_avg": 0.3243325352668762, "val/clipfrac_avg": 0.047058261930942535, "policy/entropy_avg": 0.15192937850952148, "val/ratio": 0.9890267252922058, "val/ratio_var": 7.159873348427936e-05, "val/num_eos_tokens": 93, "lr": 3.978740091051599e-06, "episode": 11648, "epoch": 0.5882828282828283, "step": 728}, {"eps": 0, "objective/kl": 34.32489013671875, "objective/entropy": -75.26506042480469, "objective/non_score_reward": -1.7162445783615112, "objective/rlhf_reward": 13.9556303024292, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008753975853323936, "policy/clipfrac_avg": 0.006253514904528856, "loss/policy_avg": 0.061702877283096313, "loss/value_avg": 0.25130608677864075, "val/clipfrac_avg": 0.03564208000898361, "policy/entropy_avg": 0.17108154296875, "val/ratio": 0.9883392453193665, "val/ratio_var": 5.439678716356866e-05, "val/num_eos_tokens": 194, "lr": 3.965668229095546e-06, "episode": 11664, "epoch": 0.5890909090909091, "step": 729}, {"eps": 0, "objective/kl": 38.28752136230469, "objective/entropy": -211.5462646484375, "objective/non_score_reward": -1.9143760204315186, "objective/rlhf_reward": 13.062187194824219, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008200224256142974, "policy/clipfrac_avg": 0.009272422641515732, "loss/policy_avg": 0.5118429064750671, "loss/value_avg": 0.3809865713119507, "val/clipfrac_avg": 0.13058164715766907, "policy/entropy_avg": 0.15618228912353516, "val/ratio": 0.9892928600311279, "val/ratio_var": 5.910802065045573e-05, "val/num_eos_tokens": 191, "lr": 3.9526037486409275e-06, "episode": 11680, "epoch": 0.5898989898989899, "step": 730}, {"eps": 0, "objective/kl": 33.536251068115234, "objective/entropy": -155.975341796875, "objective/non_score_reward": -1.6768126487731934, "objective/rlhf_reward": 14.198186874389648, "objective/scores": 15.875, "policy/approxkl_avg": 0.0006373925716616213, "policy/clipfrac_avg": 0.011547036468982697, "loss/policy_avg": -0.575583279132843, "loss/value_avg": 0.6588011384010315, "val/clipfrac_avg": 0.1309666782617569, "policy/entropy_avg": 0.15375041961669922, "val/ratio": 0.9912899732589722, "val/ratio_var": 7.494341116398573e-05, "val/num_eos_tokens": 393, "lr": 3.939546742922318e-06, "episode": 11696, "epoch": 0.5907070707070707, "step": 731}, {"eps": 0, "objective/kl": 38.771400451660156, "objective/entropy": -46.366912841796875, "objective/non_score_reward": -1.9385700225830078, "objective/rlhf_reward": 14.631742477416992, "objective/scores": 16.625, "policy/approxkl_avg": 0.0010276802349835634, "policy/clipfrac_avg": 0.004419120494276285, "loss/policy_avg": 0.08975643664598465, "loss/value_avg": 0.4319133758544922, "val/clipfrac_avg": 0.02601846680045128, "policy/entropy_avg": 0.19952774047851562, "val/ratio": 0.9856929779052734, "val/ratio_var": 5.2173218136886135e-05, "val/num_eos_tokens": 121, "lr": 3.926497305120943e-06, "episode": 11712, "epoch": 0.5915151515151515, "step": 732}, {"eps": 0, "objective/kl": 34.644386291503906, "objective/entropy": -30.65240478515625, "objective/non_score_reward": -1.7322194576263428, "objective/rlhf_reward": 16.127155303955078, "objective/scores": 17.875, "policy/approxkl_avg": 0.0009229708230122924, "policy/clipfrac_avg": 0.0053136469796299934, "loss/policy_avg": -0.5671601295471191, "loss/value_avg": 1.4009994268417358, "val/clipfrac_avg": 0.16779303550720215, "policy/entropy_avg": 0.1684565544128418, "val/ratio": 0.98807692527771, "val/ratio_var": 4.935808829031885e-05, "val/num_eos_tokens": 257, "lr": 3.913455528364027e-06, "episode": 11728, "epoch": 0.5923232323232324, "step": 733}, {"eps": 0, "objective/kl": 38.92997741699219, "objective/entropy": -115.830078125, "objective/non_score_reward": -1.9464988708496094, "objective/rlhf_reward": 12.38162612915039, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0006995117873884737, "policy/clipfrac_avg": 0.0067988173104822636, "loss/policy_avg": 0.1117314025759697, "loss/value_avg": 0.6027213335037231, "val/clipfrac_avg": 0.11902365833520889, "policy/entropy_avg": 0.12786293029785156, "val/ratio": 0.9913206696510315, "val/ratio_var": 3.536290387273766e-05, "val/num_eos_tokens": 256, "lr": 3.900421505724119e-06, "episode": 11744, "epoch": 0.5931313131313132, "step": 734}, {"eps": 0, "objective/kl": 34.79220199584961, "objective/entropy": -77.05795288085938, "objective/non_score_reward": -1.739610195159912, "objective/rlhf_reward": 12.510390281677246, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009019911522045732, "policy/clipfrac_avg": 0.00438590906560421, "loss/policy_avg": 0.07707646489143372, "loss/value_avg": 0.8557188510894775, "val/clipfrac_avg": 0.20910227298736572, "policy/entropy_avg": 0.18432998657226562, "val/ratio": 0.9879709482192993, "val/ratio_var": 7.31311592971906e-05, "val/num_eos_tokens": 339, "lr": 3.887395330218429e-06, "episode": 11760, "epoch": 0.593939393939394, "step": 735}, {"eps": 0, "objective/kl": 40.491024017333984, "objective/entropy": -189.58908081054688, "objective/non_score_reward": -2.0245513916015625, "objective/rlhf_reward": 12.608261108398438, "objective/scores": 14.625, "policy/approxkl_avg": 0.0010324950562790036, "policy/clipfrac_avg": 0.008821255527436733, "loss/policy_avg": 0.005743637681007385, "loss/value_avg": 0.4823000729084015, "val/clipfrac_avg": 0.03615584969520569, "policy/entropy_avg": 0.1864786148071289, "val/ratio": 0.9867680072784424, "val/ratio_var": 0.00012203787628095597, "val/num_eos_tokens": 146, "lr": 3.8743770948081675e-06, "episode": 11776, "epoch": 0.5947474747474748, "step": 736}, {"eps": 0, "objective/kl": 47.2681999206543, "objective/entropy": -53.57340621948242, "objective/non_score_reward": -2.363409996032715, "objective/rlhf_reward": 11.238153457641602, "objective/scores": 13.625, "policy/approxkl_avg": 0.0010416920995339751, "policy/clipfrac_avg": 0.006328020244836807, "loss/policy_avg": 0.11549818515777588, "loss/value_avg": 0.4191131591796875, "val/clipfrac_avg": 0.05454040318727493, "policy/entropy_avg": 0.21573257446289062, "val/ratio": 0.9863517880439758, "val/ratio_var": 5.17384396516718e-05, "val/num_eos_tokens": 145, "lr": 3.861366892397878e-06, "episode": 11792, "epoch": 0.5955555555555555, "step": 737}, {"eps": 0, "objective/kl": 41.81903839111328, "objective/entropy": -9.551445007324219, "objective/non_score_reward": -2.090951919555664, "objective/rlhf_reward": 15.362173080444336, "objective/scores": 17.5, "policy/approxkl_avg": 0.001077502267435193, "policy/clipfrac_avg": 0.007244379259645939, "loss/policy_avg": 0.09252296388149261, "loss/value_avg": 0.24368464946746826, "val/clipfrac_avg": 0.046727634966373444, "policy/entropy_avg": 0.208465576171875, "val/ratio": 0.9859708547592163, "val/ratio_var": 4.752089807880111e-05, "val/num_eos_tokens": 70, "lr": 3.848364815834782e-06, "episode": 11808, "epoch": 0.5963636363636363, "step": 738}, {"eps": 0, "objective/kl": 32.577239990234375, "objective/entropy": -109.4366226196289, "objective/non_score_reward": -1.6288621425628662, "objective/rlhf_reward": 13.332075119018555, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0009311667527072132, "policy/clipfrac_avg": 0.007746369112282991, "loss/policy_avg": -0.47144564986228943, "loss/value_avg": 0.9659825563430786, "val/clipfrac_avg": 0.1542586386203766, "policy/entropy_avg": 0.2024822235107422, "val/ratio": 0.9888983964920044, "val/ratio_var": 8.097994577838108e-05, "val/num_eos_tokens": 140, "lr": 3.835370957908108e-06, "episode": 11824, "epoch": 0.5971717171717171, "step": 739}, {"eps": 0, "objective/kl": 38.616180419921875, "objective/entropy": -82.36312103271484, "objective/non_score_reward": -1.9308089017868042, "objective/rlhf_reward": 12.725440979003906, "objective/scores": 14.625, "policy/approxkl_avg": 0.0008991706417873502, "policy/clipfrac_avg": 0.01843995228409767, "loss/policy_avg": 0.3934873938560486, "loss/value_avg": 0.5160516500473022, "val/clipfrac_avg": 0.0067659346386790276, "policy/entropy_avg": 0.2245502471923828, "val/ratio": 0.988348662853241, "val/ratio_var": 5.281575795379467e-05, "val/num_eos_tokens": 30, "lr": 3.822385411348429e-06, "episode": 11840, "epoch": 0.597979797979798, "step": 740}, {"eps": 0, "objective/kl": 46.57951736450195, "objective/entropy": -89.22055053710938, "objective/non_score_reward": -2.3289759159088135, "objective/rlhf_reward": 13.483524322509766, "objective/scores": 15.8125, "policy/approxkl_avg": 0.000996086047962308, "policy/clipfrac_avg": 0.005896539893001318, "loss/policy_avg": -0.06747608631849289, "loss/value_avg": 0.3114325702190399, "val/clipfrac_avg": 0.0699368417263031, "policy/entropy_avg": 0.19659423828125, "val/ratio": 0.9875314235687256, "val/ratio_var": 4.793483094545081e-05, "val/num_eos_tokens": 107, "lr": 3.809408268827009e-06, "episode": 11856, "epoch": 0.5987878787878788, "step": 741}, {"eps": 0, "objective/kl": 36.626800537109375, "objective/entropy": -104.62603759765625, "objective/non_score_reward": -1.831339955329895, "objective/rlhf_reward": 14.778034210205078, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008760478813201189, "policy/clipfrac_avg": 0.005249073728919029, "loss/policy_avg": -0.2737995684146881, "loss/value_avg": 0.4772384762763977, "val/clipfrac_avg": 0.06328597664833069, "policy/entropy_avg": 0.17531394958496094, "val/ratio": 0.9885821342468262, "val/ratio_var": 5.911157131777145e-05, "val/num_eos_tokens": 91, "lr": 3.7964396229551365e-06, "episode": 11872, "epoch": 0.5995959595959596, "step": 742}, {"eps": 0, "objective/kl": 35.787086486816406, "objective/entropy": -123.75762176513672, "objective/non_score_reward": -1.7893544435501099, "objective/rlhf_reward": 14.86689567565918, "objective/scores": 16.625, "policy/approxkl_avg": 0.0007770034717395902, "policy/clipfrac_avg": 0.0056176697835326195, "loss/policy_avg": -0.11605305969715118, "loss/value_avg": 0.30427810549736023, "val/clipfrac_avg": 0.048395052552223206, "policy/entropy_avg": 0.152313232421875, "val/ratio": 0.9905653595924377, "val/ratio_var": 5.709798642783426e-05, "val/num_eos_tokens": 386, "lr": 3.783479566283457e-06, "episode": 11888, "epoch": 0.6004040404040404, "step": 743}, {"eps": 0, "objective/kl": 43.74962615966797, "objective/entropy": -39.3416633605957, "objective/non_score_reward": -2.18748140335083, "objective/rlhf_reward": 12.648456573486328, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0010895703453570604, "policy/clipfrac_avg": 0.0052008070051670074, "loss/policy_avg": -0.02294028550386429, "loss/value_avg": 0.7253668904304504, "val/clipfrac_avg": 0.01829145848751068, "policy/entropy_avg": 0.22774124145507812, "val/ratio": 0.985338568687439, "val/ratio_var": 6.19284255662933e-05, "val/num_eos_tokens": 71, "lr": 3.7705281913013286e-06, "episode": 11904, "epoch": 0.6012121212121212, "step": 744}, {"eps": 0, "objective/kl": 33.780616760253906, "objective/entropy": -79.38550567626953, "objective/non_score_reward": -1.6890308856964111, "objective/rlhf_reward": 14.178156852722168, "objective/scores": 15.875, "policy/approxkl_avg": 0.0008661438478156924, "policy/clipfrac_avg": 0.003558866214007139, "loss/policy_avg": 0.1328388750553131, "loss/value_avg": 1.3390746116638184, "val/clipfrac_avg": 0.05471283197402954, "policy/entropy_avg": 0.15964603424072266, "val/ratio": 0.9897934198379517, "val/ratio_var": 8.215933485189453e-05, "val/num_eos_tokens": 81, "lr": 3.7575855904361447e-06, "episode": 11920, "epoch": 0.602020202020202, "step": 745}, {"eps": 0, "objective/kl": 42.908660888671875, "objective/entropy": -124.25365447998047, "objective/non_score_reward": -2.145432949066162, "objective/rlhf_reward": 11.87019157409668, "objective/scores": 14.0, "policy/approxkl_avg": 0.0009470224613323808, "policy/clipfrac_avg": 0.006878838874399662, "loss/policy_avg": -0.29737263917922974, "loss/value_avg": 0.6314270496368408, "val/clipfrac_avg": 0.0798804834485054, "policy/entropy_avg": 0.1818103790283203, "val/ratio": 0.9880188703536987, "val/ratio_var": 8.261724724434316e-05, "val/num_eos_tokens": 329, "lr": 3.7446518560526903e-06, "episode": 11936, "epoch": 0.6028282828282828, "step": 746}, {"eps": 0, "objective/kl": 43.510276794433594, "objective/entropy": -72.31340026855469, "objective/non_score_reward": -2.175513982772827, "objective/rlhf_reward": 12.347923278808594, "objective/scores": 14.5, "policy/approxkl_avg": 0.0009941391181200743, "policy/clipfrac_avg": 0.00658874586224556, "loss/policy_avg": 0.12454540282487869, "loss/value_avg": 0.4487236738204956, "val/clipfrac_avg": 0.1540203094482422, "policy/entropy_avg": 0.16569900512695312, "val/ratio": 0.9888790845870972, "val/ratio_var": 2.4041321012191474e-05, "val/num_eos_tokens": 77, "lr": 3.731727080452464e-06, "episode": 11952, "epoch": 0.6036363636363636, "step": 747}, {"eps": 0, "objective/kl": 35.28123474121094, "objective/entropy": -103.35200500488281, "objective/non_score_reward": -1.764061689376831, "objective/rlhf_reward": 13.27500057220459, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0008191847009584308, "policy/clipfrac_avg": 0.006343110464513302, "loss/policy_avg": -0.1547861397266388, "loss/value_avg": 0.49022629857063293, "val/clipfrac_avg": 0.04430587589740753, "policy/entropy_avg": 0.14904117584228516, "val/ratio": 0.9908789396286011, "val/ratio_var": 4.545380579656921e-05, "val/num_eos_tokens": 124, "lr": 3.7188113558730403e-06, "episode": 11968, "epoch": 0.6044444444444445, "step": 748}, {"eps": 0, "objective/kl": 36.76349639892578, "objective/entropy": -111.66493225097656, "objective/non_score_reward": -1.838174819946289, "objective/rlhf_reward": 13.693075180053711, "objective/scores": 15.5, "policy/approxkl_avg": 0.0008918631356209517, "policy/clipfrac_avg": 0.010910662822425365, "loss/policy_avg": 0.11601711809635162, "loss/value_avg": 0.6175556182861328, "val/clipfrac_avg": 0.06755826622247696, "policy/entropy_avg": 0.16454505920410156, "val/ratio": 0.9886500835418701, "val/ratio_var": 6.255816697375849e-05, "val/num_eos_tokens": 358, "lr": 3.705904774487396e-06, "episode": 11984, "epoch": 0.6052525252525253, "step": 749}, {"eps": 0, "objective/kl": 32.27269744873047, "objective/entropy": -63.2913703918457, "objective/non_score_reward": -1.6136349439620972, "objective/rlhf_reward": 14.644177436828613, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009490256779827178, "policy/clipfrac_avg": 0.0068245502188801765, "loss/policy_avg": -0.04627024754881859, "loss/value_avg": 0.6224689483642578, "val/clipfrac_avg": 0.18630267679691315, "policy/entropy_avg": 0.17229938507080078, "val/ratio": 0.9878020882606506, "val/ratio_var": 5.455148493638262e-05, "val/num_eos_tokens": 11, "lr": 3.6930074284032613e-06, "episode": 12000, "epoch": 0.6060606060606061, "step": 750}, {"eps": 0, "objective/kl": 34.44614028930664, "objective/entropy": -54.77946472167969, "objective/non_score_reward": -1.7223069667816162, "objective/rlhf_reward": 14.894880294799805, "objective/scores": 16.625, "policy/approxkl_avg": 0.0011779924388974905, "policy/clipfrac_avg": 0.024402525275945663, "loss/policy_avg": -0.17890788614749908, "loss/value_avg": 0.7140947580337524, "val/clipfrac_avg": 0.04182715713977814, "policy/entropy_avg": 0.2098255157470703, "val/ratio": 0.9854939579963684, "val/ratio_var": 0.00014133901277091354, "val/num_eos_tokens": 42, "lr": 3.6801194096624515e-06, "episode": 12016, "epoch": 0.6068686868686869, "step": 751}, {"eps": 0, "objective/kl": 37.67994689941406, "objective/entropy": -70.776611328125, "objective/non_score_reward": -1.8839974403381348, "objective/rlhf_reward": 12.108190536499023, "objective/scores": 14.0, "policy/approxkl_avg": 0.0009373503271490335, "policy/clipfrac_avg": 0.005210112780332565, "loss/policy_avg": -0.0015007667243480682, "loss/value_avg": 0.2544102668762207, "val/clipfrac_avg": 0.021128622815012932, "policy/entropy_avg": 0.18710041046142578, "val/ratio": 0.9874061942100525, "val/ratio_var": 5.212551332078874e-05, "val/num_eos_tokens": 218, "lr": 3.6672408102402247e-06, "episode": 12032, "epoch": 0.6076767676767677, "step": 752}, {"eps": 0, "objective/kl": 44.5589714050293, "objective/entropy": -89.17417907714844, "objective/non_score_reward": -2.2279486656188965, "objective/rlhf_reward": 12.178300857543945, "objective/scores": 14.375, "policy/approxkl_avg": 0.0009375615045428276, "policy/clipfrac_avg": 0.005882620811462402, "loss/policy_avg": 0.05351484939455986, "loss/value_avg": 0.28849858045578003, "val/clipfrac_avg": 0.021566759794950485, "policy/entropy_avg": 0.18878555297851562, "val/ratio": 0.9880841970443726, "val/ratio_var": 5.727734605898149e-05, "val/num_eos_tokens": 120, "lr": 3.654371722044616e-06, "episode": 12048, "epoch": 0.6084848484848485, "step": 753}, {"eps": 0, "objective/kl": 43.28697967529297, "objective/entropy": -114.3979263305664, "objective/non_score_reward": -2.16434907913208, "objective/rlhf_reward": 14.226276397705078, "objective/scores": 16.375, "policy/approxkl_avg": 0.000866133370436728, "policy/clipfrac_avg": 0.0050315093249082565, "loss/policy_avg": -0.12424173951148987, "loss/value_avg": 0.359388530254364, "val/clipfrac_avg": 0.1426960825920105, "policy/entropy_avg": 0.16837501525878906, "val/ratio": 0.9894628524780273, "val/ratio_var": 5.016153590986505e-05, "val/num_eos_tokens": 391, "lr": 3.641512236915783e-06, "episode": 12064, "epoch": 0.6092929292929293, "step": 754}, {"eps": 0, "objective/kl": 38.83340072631836, "objective/entropy": -164.57290649414062, "objective/non_score_reward": -1.9416700601577759, "objective/rlhf_reward": 14.245829582214355, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008140793070197105, "policy/clipfrac_avg": 0.00435154652222991, "loss/policy_avg": -0.2821425795555115, "loss/value_avg": 0.38819342851638794, "val/clipfrac_avg": 0.09525477886199951, "policy/entropy_avg": 0.15729975700378418, "val/ratio": 0.989446222782135, "val/ratio_var": 5.3185020078672096e-05, "val/num_eos_tokens": 336, "lr": 3.6286624466253496e-06, "episode": 12080, "epoch": 0.6101010101010101, "step": 755}, {"eps": 0, "objective/kl": 32.80990219116211, "objective/entropy": -61.176116943359375, "objective/non_score_reward": -1.6404950618743896, "objective/rlhf_reward": 16.24231719970703, "objective/scores": 17.875, "policy/approxkl_avg": 0.0009597806492820382, "policy/clipfrac_avg": 0.0061972904950380325, "loss/policy_avg": 0.05094696208834648, "loss/value_avg": 0.4260476529598236, "val/clipfrac_avg": 0.01815173029899597, "policy/entropy_avg": 0.16566753387451172, "val/ratio": 0.9894044399261475, "val/ratio_var": 4.711053770733997e-05, "val/num_eos_tokens": 163, "lr": 3.6158224428757538e-06, "episode": 12096, "epoch": 0.610909090909091, "step": 756}, {"eps": 0, "objective/kl": 34.218990325927734, "objective/entropy": -52.30070877075195, "objective/non_score_reward": -1.7109496593475342, "objective/rlhf_reward": 13.273425102233887, "objective/scores": 15.0, "policy/approxkl_avg": 0.0009450222714804113, "policy/clipfrac_avg": 0.0049097794108092785, "loss/policy_avg": -0.040469832718372345, "loss/value_avg": 0.3746616244316101, "val/clipfrac_avg": 0.09231511503458023, "policy/entropy_avg": 0.1805877685546875, "val/ratio": 0.9877369403839111, "val/ratio_var": 4.407649248605594e-05, "val/num_eos_tokens": 105, "lr": 3.602992317299594e-06, "episode": 12112, "epoch": 0.6117171717171718, "step": 757}, {"eps": 0, "objective/kl": 42.89472961425781, "objective/entropy": -33.08611297607422, "objective/non_score_reward": -2.1447365283966064, "objective/rlhf_reward": 14.792763710021973, "objective/scores": 17.0, "policy/approxkl_avg": 0.0011628612410277128, "policy/clipfrac_avg": 0.00601824838668108, "loss/policy_avg": -0.0790981650352478, "loss/value_avg": 0.289002388715744, "val/clipfrac_avg": 0.05997534096240997, "policy/entropy_avg": 0.21850204467773438, "val/ratio": 0.9858998656272888, "val/ratio_var": 4.6025765186641365e-05, "val/num_eos_tokens": 208, "lr": 3.5901721614589725e-06, "episode": 12128, "epoch": 0.6125252525252525, "step": 758}, {"eps": 0, "objective/kl": 34.86107635498047, "objective/entropy": -57.002418518066406, "objective/non_score_reward": -1.743053674697876, "objective/rlhf_reward": 12.421009063720703, "objective/scores": 14.1875, "policy/approxkl_avg": 0.0009696290362626314, "policy/clipfrac_avg": 0.005794525146484375, "loss/policy_avg": 0.08223091810941696, "loss/value_avg": 0.4521709978580475, "val/clipfrac_avg": 0.03934604302048683, "policy/entropy_avg": 0.1826324462890625, "val/ratio": 0.9880379438400269, "val/ratio_var": 4.820780668524094e-05, "val/num_eos_tokens": 137, "lr": 3.5773620668448384e-06, "episode": 12144, "epoch": 0.6133333333333333, "step": 759}, {"eps": 0, "objective/kl": 40.15751647949219, "objective/entropy": -58.77183532714844, "objective/non_score_reward": -2.007875919342041, "objective/rlhf_reward": 13.171812057495117, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0009755170904099941, "policy/clipfrac_avg": 0.0075699929147958755, "loss/policy_avg": 0.012903919443488121, "loss/value_avg": 0.22128555178642273, "val/clipfrac_avg": 0.007695037871599197, "policy/entropy_avg": 0.1793661117553711, "val/ratio": 0.9879107475280762, "val/ratio_var": 9.122795017901808e-05, "val/num_eos_tokens": 194, "lr": 3.564562124876344e-06, "episode": 12160, "epoch": 0.6141414141414141, "step": 760}, {"eps": 0, "objective/kl": 36.40907287597656, "objective/entropy": -92.11294555664062, "objective/non_score_reward": -1.8204538822174072, "objective/rlhf_reward": 13.468608856201172, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0009949133964255452, "policy/clipfrac_avg": 0.012031201273202896, "loss/policy_avg": 0.23269692063331604, "loss/value_avg": 0.6034168004989624, "val/clipfrac_avg": 0.03142271935939789, "policy/entropy_avg": 0.19497203826904297, "val/ratio": 0.9871051907539368, "val/ratio_var": 7.474326412193477e-05, "val/num_eos_tokens": 210, "lr": 3.5517724269001853e-06, "episode": 12176, "epoch": 0.6149494949494949, "step": 761}, {"eps": 0, "objective/kl": 32.94031524658203, "objective/entropy": -42.83490753173828, "objective/non_score_reward": -1.6470158100128174, "objective/rlhf_reward": 13.567828178405762, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0011591427028179169, "policy/clipfrac_avg": 0.004484353121370077, "loss/policy_avg": -0.014487015083432198, "loss/value_avg": 1.0809375047683716, "val/clipfrac_avg": 0.1017560362815857, "policy/entropy_avg": 0.21469688415527344, "val/ratio": 0.9864501953125, "val/ratio_var": 8.947903552325442e-05, "val/num_eos_tokens": 453, "lr": 3.538993064189954e-06, "episode": 12192, "epoch": 0.6157575757575757, "step": 762}, {"eps": 0, "objective/kl": 31.28769302368164, "objective/entropy": -71.51974487304688, "objective/non_score_reward": -1.5643846988677979, "objective/rlhf_reward": 15.216865539550781, "objective/scores": 16.75, "policy/approxkl_avg": 0.0007811100222170353, "policy/clipfrac_avg": 0.0053521376103162766, "loss/policy_avg": -0.0410991832613945, "loss/value_avg": 1.3014378547668457, "val/clipfrac_avg": 0.059704042971134186, "policy/entropy_avg": 0.16764509677886963, "val/ratio": 0.9894566535949707, "val/ratio_var": 6.841208232799545e-05, "val/num_eos_tokens": 203, "lr": 3.526224127945479e-06, "episode": 12208, "epoch": 0.6165656565656565, "step": 763}, {"eps": 0, "objective/kl": 41.09782028198242, "objective/entropy": -99.9854736328125, "objective/non_score_reward": -2.0548908710479736, "objective/rlhf_reward": 10.624795913696289, "objective/scores": 12.6875, "policy/approxkl_avg": 0.0009035035036504269, "policy/clipfrac_avg": 0.010427422821521759, "loss/policy_avg": 0.112126424908638, "loss/value_avg": 0.725554347038269, "val/clipfrac_avg": 0.12764863669872284, "policy/entropy_avg": 0.17998504638671875, "val/ratio": 0.988477349281311, "val/ratio_var": 8.909796451916918e-05, "val/num_eos_tokens": 226, "lr": 3.5134657092921877e-06, "episode": 12224, "epoch": 0.6173737373737374, "step": 764}, {"eps": 0, "objective/kl": 41.466583251953125, "objective/entropy": -135.23390197753906, "objective/non_score_reward": -2.073329210281372, "objective/rlhf_reward": 12.94229507446289, "objective/scores": 15.0, "policy/approxkl_avg": 0.0009621024364605546, "policy/clipfrac_avg": 0.00628144945949316, "loss/policy_avg": 0.37758830189704895, "loss/value_avg": 0.7465800046920776, "val/clipfrac_avg": 0.10096652805805206, "policy/entropy_avg": 0.17942047119140625, "val/ratio": 0.9877568483352661, "val/ratio_var": 7.067532715154812e-05, "val/num_eos_tokens": 233, "lr": 3.500717899280442e-06, "episode": 12240, "epoch": 0.6181818181818182, "step": 765}, {"eps": 0, "objective/kl": 39.61420440673828, "objective/entropy": -86.19502258300781, "objective/non_score_reward": -1.9807100296020508, "objective/rlhf_reward": 13.706789016723633, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0009516358841210604, "policy/clipfrac_avg": 0.01008635200560093, "loss/policy_avg": -0.10782650113105774, "loss/value_avg": 0.4996359944343567, "val/clipfrac_avg": 0.05575938522815704, "policy/entropy_avg": 0.17633438110351562, "val/ratio": 0.9884860515594482, "val/ratio_var": 5.413329563452862e-05, "val/num_eos_tokens": 426, "lr": 3.4879807888849026e-06, "episode": 12256, "epoch": 0.618989898989899, "step": 766}, {"eps": 0, "objective/kl": 38.48719787597656, "objective/entropy": -78.26728057861328, "objective/non_score_reward": -1.9243600368499756, "objective/rlhf_reward": 11.661577224731445, "objective/scores": 13.5625, "policy/approxkl_avg": 0.0009466722258366644, "policy/clipfrac_avg": 0.0042708818800747395, "loss/policy_avg": 0.0611935630440712, "loss/value_avg": 0.40072089433670044, "val/clipfrac_avg": 0.1151452288031578, "policy/entropy_avg": 0.17498779296875, "val/ratio": 0.9874659776687622, "val/ratio_var": 7.443072536261752e-05, "val/num_eos_tokens": 415, "lr": 3.475254469003865e-06, "episode": 12272, "epoch": 0.6197979797979798, "step": 767}, {"eps": 0, "objective/kl": 41.06260681152344, "objective/entropy": -45.31693649291992, "objective/non_score_reward": -2.0531303882598877, "objective/rlhf_reward": 13.860933303833008, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0011651598615571856, "policy/clipfrac_avg": 0.0047182347625494, "loss/policy_avg": -0.07818017899990082, "loss/value_avg": 0.588293731212616, "val/clipfrac_avg": 0.16338574886322021, "policy/entropy_avg": 0.2090606689453125, "val/ratio": 0.986497700214386, "val/ratio_var": 8.175932453013957e-05, "val/num_eos_tokens": 9, "lr": 3.4625390304586224e-06, "episode": 12288, "epoch": 0.6206060606060606, "step": 768}, {"eps": 0, "objective/kl": 35.794551849365234, "objective/entropy": -103.96510314941406, "objective/non_score_reward": -1.7897276878356934, "objective/rlhf_reward": 13.507146835327148, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0010084104724228382, "policy/clipfrac_avg": 0.0070042358711361885, "loss/policy_avg": -0.3265213966369629, "loss/value_avg": 0.5878744125366211, "val/clipfrac_avg": 0.12878292798995972, "policy/entropy_avg": 0.19075870513916016, "val/ratio": 0.9861502051353455, "val/ratio_var": 8.973718649940565e-05, "val/num_eos_tokens": 8, "lr": 3.4498345639928167e-06, "episode": 12304, "epoch": 0.6214141414141414, "step": 769}, {"eps": 0, "objective/kl": 31.87890625, "objective/entropy": -92.59934997558594, "objective/non_score_reward": -1.5939452648162842, "objective/rlhf_reward": 14.429492950439453, "objective/scores": 16.0, "policy/approxkl_avg": 0.0007836055592633784, "policy/clipfrac_avg": 0.004061062820255756, "loss/policy_avg": -0.24864813685417175, "loss/value_avg": 0.703667402267456, "val/clipfrac_avg": 0.14356926083564758, "policy/entropy_avg": 0.13613224029541016, "val/ratio": 0.9909623265266418, "val/ratio_var": 6.252773891901597e-05, "val/num_eos_tokens": 22, "lr": 3.4371411602717785e-06, "episode": 12320, "epoch": 0.6222222222222222, "step": 770}, {"eps": 0, "objective/kl": 32.05082321166992, "objective/entropy": -201.825927734375, "objective/non_score_reward": -1.602541208267212, "objective/rlhf_reward": 13.842771530151367, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0009224842651747167, "policy/clipfrac_avg": 0.004090993665158749, "loss/policy_avg": -0.11106158792972565, "loss/value_avg": 0.7719787359237671, "val/clipfrac_avg": 0.06381858140230179, "policy/entropy_avg": 0.19452500343322754, "val/ratio": 0.9873221516609192, "val/ratio_var": 0.00010800129530252889, "val/num_eos_tokens": 137, "lr": 3.424458909881897e-06, "episode": 12336, "epoch": 0.623030303030303, "step": 771}, {"eps": 0, "objective/kl": 28.94087791442871, "objective/entropy": -97.80130767822266, "objective/non_score_reward": -1.4470438957214355, "objective/rlhf_reward": 13.572486877441406, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008233180269598961, "policy/clipfrac_avg": 0.007523335050791502, "loss/policy_avg": -0.3147702217102051, "loss/value_avg": 1.0198739767074585, "val/clipfrac_avg": 0.021688329055905342, "policy/entropy_avg": 0.17452716827392578, "val/ratio": 0.9885031580924988, "val/ratio_var": 6.793635111534968e-05, "val/num_eos_tokens": 116, "lr": 3.411787903329963e-06, "episode": 12352, "epoch": 0.6238383838383839, "step": 772}, {"eps": 0, "objective/kl": 33.001129150390625, "objective/entropy": -61.794166564941406, "objective/non_score_reward": -1.6500564813613892, "objective/rlhf_reward": 12.943693161010742, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009417567634955049, "policy/clipfrac_avg": 0.005282199941575527, "loss/policy_avg": -0.019919998943805695, "loss/value_avg": 0.31691867113113403, "val/clipfrac_avg": 0.05137361213564873, "policy/entropy_avg": 0.1732311248779297, "val/ratio": 0.9889352321624756, "val/ratio_var": 3.9570189983351156e-05, "val/num_eos_tokens": 13, "lr": 3.3991282310425276e-06, "episode": 12368, "epoch": 0.6246464646464647, "step": 773}, {"eps": 0, "objective/kl": 29.43255615234375, "objective/entropy": -73.91504669189453, "objective/non_score_reward": -1.4716278314590454, "objective/rlhf_reward": 13.293996810913086, "objective/scores": 14.75, "policy/approxkl_avg": 0.0010180522222071886, "policy/clipfrac_avg": 0.0050889430567622185, "loss/policy_avg": 0.010051839053630829, "loss/value_avg": 0.5414019227027893, "val/clipfrac_avg": 0.041913747787475586, "policy/entropy_avg": 0.2153911590576172, "val/ratio": 0.986315906047821, "val/ratio_var": 8.102346328087151e-05, "val/num_eos_tokens": 214, "lr": 3.386479983365251e-06, "episode": 12384, "epoch": 0.6254545454545455, "step": 774}, {"eps": 0, "objective/kl": 31.846160888671875, "objective/entropy": -75.78406524658203, "objective/non_score_reward": -1.5923081636428833, "objective/rlhf_reward": 11.946754455566406, "objective/scores": 13.5625, "policy/approxkl_avg": 0.0009044273756444454, "policy/clipfrac_avg": 0.005002772435545921, "loss/policy_avg": -0.040474407374858856, "loss/value_avg": 0.30427631735801697, "val/clipfrac_avg": 0.04153668135404587, "policy/entropy_avg": 0.18775272369384766, "val/ratio": 0.9877529144287109, "val/ratio_var": 5.462392800836824e-05, "val/num_eos_tokens": 243, "lr": 3.3738432505622653e-06, "episode": 12400, "epoch": 0.6262626262626263, "step": 775}, {"eps": 0, "objective/kl": 35.974266052246094, "objective/entropy": -97.84140014648438, "objective/non_score_reward": -1.798713207244873, "objective/rlhf_reward": 13.685661315917969, "objective/scores": 15.5, "policy/approxkl_avg": 0.0009363207500427961, "policy/clipfrac_avg": 0.0035052683670073748, "loss/policy_avg": -0.25257226824760437, "loss/value_avg": 0.47636741399765015, "val/clipfrac_avg": 0.07260937243700027, "policy/entropy_avg": 0.18935298919677734, "val/ratio": 0.9877586364746094, "val/ratio_var": 0.00010292992374161258, "val/num_eos_tokens": 26, "lr": 3.3612181228155237e-06, "episode": 12416, "epoch": 0.6270707070707071, "step": 776}, {"eps": 0, "objective/kl": 29.197486877441406, "objective/entropy": -40.24072265625, "objective/non_score_reward": -1.4598743915557861, "objective/rlhf_reward": 12.883874893188477, "objective/scores": 14.375, "policy/approxkl_avg": 0.0010831760009750724, "policy/clipfrac_avg": 0.02700810506939888, "loss/policy_avg": 0.41764986515045166, "loss/value_avg": 0.8629061579704285, "val/clipfrac_avg": 0.08423963189125061, "policy/entropy_avg": 0.16212844848632812, "val/ratio": 0.9885735511779785, "val/ratio_var": 8.724266081117094e-05, "val/num_eos_tokens": 192, "lr": 3.3486046902241663e-06, "episode": 12432, "epoch": 0.6278787878787879, "step": 777}, {"eps": 0, "objective/kl": 33.57006072998047, "objective/entropy": -69.478271484375, "objective/non_score_reward": -1.6785030364990234, "objective/rlhf_reward": 14.055871963500977, "objective/scores": 15.75, "policy/approxkl_avg": 0.0008899248205125332, "policy/clipfrac_avg": 0.006825867109000683, "loss/policy_avg": 0.2242026925086975, "loss/value_avg": 0.9421164989471436, "val/clipfrac_avg": 0.1501787304878235, "policy/entropy_avg": 0.14767837524414062, "val/ratio": 0.9905418753623962, "val/ratio_var": 4.142183388466947e-05, "val/num_eos_tokens": 109, "lr": 3.336003042803861e-06, "episode": 12448, "epoch": 0.6286868686868687, "step": 778}, {"eps": 0, "objective/kl": 31.944690704345703, "objective/entropy": -182.4083251953125, "objective/non_score_reward": -1.5972344875335693, "objective/rlhf_reward": 12.043390274047852, "objective/scores": 13.625, "policy/approxkl_avg": 0.000871892087161541, "policy/clipfrac_avg": 0.004613575059920549, "loss/policy_avg": 0.03614666312932968, "loss/value_avg": 0.7387021780014038, "val/clipfrac_avg": 0.1070820540189743, "policy/entropy_avg": 0.1733226776123047, "val/ratio": 0.9880519509315491, "val/ratio_var": 9.137512097368017e-05, "val/num_eos_tokens": 28, "lr": 3.3234132704861786e-06, "episode": 12464, "epoch": 0.6294949494949494, "step": 779}, {"eps": 0, "objective/kl": 22.97904396057129, "objective/entropy": -80.43102264404297, "objective/non_score_reward": -1.1489522457122803, "objective/rlhf_reward": 13.28073501586914, "objective/scores": 14.4375, "policy/approxkl_avg": 0.000753985601477325, "policy/clipfrac_avg": 0.013304230757057667, "loss/policy_avg": 0.491243839263916, "loss/value_avg": 1.7449349164962769, "val/clipfrac_avg": 0.12828537821769714, "policy/entropy_avg": 0.16235482692718506, "val/ratio": 0.991666316986084, "val/ratio_var": 4.57862151961308e-05, "val/num_eos_tokens": 261, "lr": 3.310835463117942e-06, "episode": 12480, "epoch": 0.6303030303030303, "step": 780}, {"eps": 0, "objective/kl": 37.96983337402344, "objective/entropy": -96.58609008789062, "objective/non_score_reward": -1.898491621017456, "objective/rlhf_reward": 12.742133140563965, "objective/scores": 14.625, "policy/approxkl_avg": 0.0008919077808968723, "policy/clipfrac_avg": 0.004138793330639601, "loss/policy_avg": -0.02106083557009697, "loss/value_avg": 0.26465463638305664, "val/clipfrac_avg": 0.022776879370212555, "policy/entropy_avg": 0.176055908203125, "val/ratio": 0.9880313873291016, "val/ratio_var": 5.30094766872935e-05, "val/num_eos_tokens": 232, "lr": 3.2982697104605855e-06, "episode": 12496, "epoch": 0.6311111111111111, "step": 781}, {"eps": 0, "objective/kl": 30.898021697998047, "objective/entropy": -113.97618103027344, "objective/non_score_reward": -1.5449011325836182, "objective/rlhf_reward": 15.892599105834961, "objective/scores": 17.5, "policy/approxkl_avg": 0.0008122409926727414, "policy/clipfrac_avg": 0.007180417887866497, "loss/policy_avg": -0.21595685184001923, "loss/value_avg": 0.1668311059474945, "val/clipfrac_avg": 0.04710119590163231, "policy/entropy_avg": 0.15228557586669922, "val/ratio": 0.9905120134353638, "val/ratio_var": 4.6862616727594286e-05, "val/num_eos_tokens": 88, "lr": 3.2857161021895125e-06, "episode": 12512, "epoch": 0.6319191919191919, "step": 782}, {"eps": 0, "objective/kl": 35.46149444580078, "objective/entropy": -119.84375762939453, "objective/non_score_reward": -1.773074746131897, "objective/rlhf_reward": 13.00036334991455, "objective/scores": 14.75, "policy/approxkl_avg": 0.000846280949190259, "policy/clipfrac_avg": 0.006307992152869701, "loss/policy_avg": -0.02830284647643566, "loss/value_avg": 0.470698744058609, "val/clipfrac_avg": 0.09303553402423859, "policy/entropy_avg": 0.15985870361328125, "val/ratio": 0.9886760711669922, "val/ratio_var": 8.341170905623585e-05, "val/num_eos_tokens": 242, "lr": 3.273174727893463e-06, "episode": 12528, "epoch": 0.6327272727272727, "step": 783}, {"eps": 0, "objective/kl": 35.053001403808594, "objective/entropy": -32.83402633666992, "objective/non_score_reward": -1.7526500225067139, "objective/rlhf_reward": 13.708287239074707, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0010523913661018014, "policy/clipfrac_avg": 0.005916148889809847, "loss/policy_avg": -0.03483296558260918, "loss/value_avg": 0.2278536707162857, "val/clipfrac_avg": 0.007893375121057034, "policy/entropy_avg": 0.2171039581298828, "val/ratio": 0.9859672784805298, "val/ratio_var": 5.139111090102233e-05, "val/num_eos_tokens": 463, "lr": 3.2606456770738636e-06, "episode": 12544, "epoch": 0.6335353535353535, "step": 784}, {"eps": 0, "objective/kl": 41.38114929199219, "objective/entropy": -109.50154113769531, "objective/non_score_reward": -2.0690574645996094, "objective/rlhf_reward": 13.82156753540039, "objective/scores": 15.875, "policy/approxkl_avg": 0.0009320294484496117, "policy/clipfrac_avg": 0.005337849725037813, "loss/policy_avg": 0.17171788215637207, "loss/value_avg": 0.5937256217002869, "val/clipfrac_avg": 0.0434221550822258, "policy/entropy_avg": 0.1828467845916748, "val/ratio": 0.9880971908569336, "val/ratio_var": 7.803420157870278e-05, "val/num_eos_tokens": 252, "lr": 3.2481290391442034e-06, "episode": 12560, "epoch": 0.6343434343434343, "step": 785}, {"eps": 0, "objective/kl": 39.33965301513672, "objective/entropy": -80.62982940673828, "objective/non_score_reward": -1.9669828414916992, "objective/rlhf_reward": 14.3767671585083, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009726607240736485, "policy/clipfrac_avg": 0.005861825309693813, "loss/policy_avg": -0.12326152622699738, "loss/value_avg": 0.22453081607818604, "val/clipfrac_avg": 0.04197417199611664, "policy/entropy_avg": 0.18674850463867188, "val/ratio": 0.9867269396781921, "val/ratio_var": 3.261497477069497e-05, "val/num_eos_tokens": 55, "lr": 3.235624903429374e-06, "episode": 12576, "epoch": 0.6351515151515151, "step": 786}, {"eps": 0, "objective/kl": 46.58558654785156, "objective/entropy": -64.31414031982422, "objective/non_score_reward": -2.3292794227600098, "objective/rlhf_reward": 14.576970100402832, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009546708315610886, "policy/clipfrac_avg": 0.00552853848785162, "loss/policy_avg": 0.02891068160533905, "loss/value_avg": 0.2414318025112152, "val/clipfrac_avg": 0.0013757861452177167, "policy/entropy_avg": 0.189300537109375, "val/ratio": 0.987205982208252, "val/ratio_var": 3.5099023079965264e-05, "val/num_eos_tokens": 363, "lr": 3.2231333591650567e-06, "episode": 12592, "epoch": 0.635959595959596, "step": 787}, {"eps": 0, "objective/kl": 42.85956573486328, "objective/entropy": -57.81846237182617, "objective/non_score_reward": -2.1429784297943115, "objective/rlhf_reward": 12.14999008178711, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0011135776294395328, "policy/clipfrac_avg": 0.009378955699503422, "loss/policy_avg": -0.19538097083568573, "loss/value_avg": 0.7160595655441284, "val/clipfrac_avg": 0.1690627932548523, "policy/entropy_avg": 0.2034454345703125, "val/ratio": 0.9861621856689453, "val/ratio_var": 9.24188134376891e-05, "val/num_eos_tokens": 481, "lr": 3.210654495497068e-06, "episode": 12608, "epoch": 0.6367676767676768, "step": 788}, {"eps": 0, "objective/kl": 36.849369049072266, "objective/entropy": -76.67124938964844, "objective/non_score_reward": -1.8424686193466187, "objective/rlhf_reward": 12.25128173828125, "objective/scores": 14.125, "policy/approxkl_avg": 0.0009952555410563946, "policy/clipfrac_avg": 0.004150367341935635, "loss/policy_avg": 0.048018310219049454, "loss/value_avg": 0.4312187135219574, "val/clipfrac_avg": 0.08804323524236679, "policy/entropy_avg": 0.18071389198303223, "val/ratio": 0.9867403507232666, "val/ratio_var": 0.00013288675108924508, "val/num_eos_tokens": 357, "lr": 3.198188401480734e-06, "episode": 12624, "epoch": 0.6375757575757576, "step": 789}, {"eps": 0, "objective/kl": 33.96638488769531, "objective/entropy": -126.28926086425781, "objective/non_score_reward": -1.6983189582824707, "objective/rlhf_reward": 12.973556518554688, "objective/scores": 14.6875, "policy/approxkl_avg": 0.0007302669691853225, "policy/clipfrac_avg": 0.008397979661822319, "loss/policy_avg": 0.18971526622772217, "loss/value_avg": 0.6626036167144775, "val/clipfrac_avg": 0.06916166841983795, "policy/entropy_avg": 0.14786386489868164, "val/ratio": 0.9910507798194885, "val/ratio_var": 5.283964492264204e-05, "val/num_eos_tokens": 600, "lr": 3.185735166080244e-06, "episode": 12640, "epoch": 0.6383838383838384, "step": 790}, {"eps": 0, "objective/kl": 40.16947555541992, "objective/entropy": -75.64164733886719, "objective/non_score_reward": -2.0084738731384277, "objective/rlhf_reward": 12.86652660369873, "objective/scores": 14.875, "policy/approxkl_avg": 0.0008897786610759795, "policy/clipfrac_avg": 0.004540015943348408, "loss/policy_avg": -0.4279332160949707, "loss/value_avg": 0.5117957592010498, "val/clipfrac_avg": 0.0338541679084301, "policy/entropy_avg": 0.18616324663162231, "val/ratio": 0.9873502254486084, "val/ratio_var": 6.438903801608831e-05, "val/num_eos_tokens": 224, "lr": 3.173294878168025e-06, "episode": 12656, "epoch": 0.6391919191919192, "step": 791}, {"eps": 0, "objective/kl": 35.08636474609375, "objective/entropy": -36.1086540222168, "objective/non_score_reward": -1.7543182373046875, "objective/rlhf_reward": 14.995681762695312, "objective/scores": 16.75, "policy/approxkl_avg": 0.0010164354462176561, "policy/clipfrac_avg": 0.0033426620066165924, "loss/policy_avg": -0.30223172903060913, "loss/value_avg": 0.7414761185646057, "val/clipfrac_avg": 0.08375494927167892, "policy/entropy_avg": 0.20384597778320312, "val/ratio": 0.9862762093544006, "val/ratio_var": 8.216432615881786e-05, "val/num_eos_tokens": 121, "lr": 3.160867626524107e-06, "episode": 12672, "epoch": 0.64, "step": 792}, {"eps": 0, "objective/kl": 33.01978302001953, "objective/entropy": -52.89218521118164, "objective/non_score_reward": -1.650989294052124, "objective/rlhf_reward": 12.270885467529297, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0009637182811275125, "policy/clipfrac_avg": 0.006595691200345755, "loss/policy_avg": 0.11556120216846466, "loss/value_avg": 0.44696569442749023, "val/clipfrac_avg": 0.0313958078622818, "policy/entropy_avg": 0.18551349639892578, "val/ratio": 0.987555742263794, "val/ratio_var": 4.9779242544900626e-05, "val/num_eos_tokens": 483, "lr": 3.1484534998354832e-06, "episode": 12688, "epoch": 0.6408080808080808, "step": 793}, {"eps": 0, "objective/kl": 38.74113845825195, "objective/entropy": -44.2855110168457, "objective/non_score_reward": -1.9370570182800293, "objective/rlhf_reward": 13.531692504882812, "objective/scores": 15.5, "policy/approxkl_avg": 0.0009892593370750546, "policy/clipfrac_avg": 0.006002710200846195, "loss/policy_avg": -0.11985203623771667, "loss/value_avg": 0.5595318078994751, "val/clipfrac_avg": 0.051367055624723434, "policy/entropy_avg": 0.20067405700683594, "val/ratio": 0.98707115650177, "val/ratio_var": 6.210069841472432e-05, "val/num_eos_tokens": 146, "lr": 3.136052586695481e-06, "episode": 12704, "epoch": 0.6416161616161616, "step": 794}, {"eps": 0, "objective/kl": 36.60810852050781, "objective/entropy": -47.063812255859375, "objective/non_score_reward": -1.8304054737091064, "objective/rlhf_reward": 13.661782264709473, "objective/scores": 15.5, "policy/approxkl_avg": 0.000998336123302579, "policy/clipfrac_avg": 0.004183639772236347, "loss/policy_avg": 0.06161116436123848, "loss/value_avg": 0.20387178659439087, "val/clipfrac_avg": 0.08834637701511383, "policy/entropy_avg": 0.19140625, "val/ratio": 0.9874628782272339, "val/ratio_var": 3.3792719477787614e-05, "val/num_eos_tokens": 342, "lr": 3.12366497560313e-06, "episode": 12720, "epoch": 0.6424242424242425, "step": 795}, {"eps": 0, "objective/kl": 35.12114715576172, "objective/entropy": -104.79203796386719, "objective/non_score_reward": -1.7560575008392334, "objective/rlhf_reward": 13.486129760742188, "objective/scores": 15.25, "policy/approxkl_avg": 0.0008146626059897244, "policy/clipfrac_avg": 0.0056906514801084995, "loss/policy_avg": 0.02652081847190857, "loss/value_avg": 0.4432865381240845, "val/clipfrac_avg": 0.061473242938518524, "policy/entropy_avg": 0.1502237319946289, "val/ratio": 0.9898946285247803, "val/ratio_var": 4.619114042725414e-05, "val/num_eos_tokens": 748, "lr": 3.1112907549625304e-06, "episode": 12736, "epoch": 0.6432323232323233, "step": 796}, {"eps": 0, "objective/kl": 38.224117279052734, "objective/entropy": -88.31727600097656, "objective/non_score_reward": -1.9112060070037842, "objective/rlhf_reward": 14.698169708251953, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008200570591725409, "policy/clipfrac_avg": 0.007693252060562372, "loss/policy_avg": 0.06834741681814194, "loss/value_avg": 0.5731651782989502, "val/clipfrac_avg": 0.11237858235836029, "policy/entropy_avg": 0.1607208251953125, "val/ratio": 0.9905504584312439, "val/ratio_var": 4.706854815594852e-05, "val/num_eos_tokens": 240, "lr": 3.098930013082223e-06, "episode": 12752, "epoch": 0.6440404040404041, "step": 797}, {"eps": 0, "objective/kl": 29.702392578125, "objective/entropy": -124.7913589477539, "objective/non_score_reward": -1.4851195812225342, "objective/rlhf_reward": 13.68675422668457, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0008782441727817059, "policy/clipfrac_avg": 0.00598434591665864, "loss/policy_avg": -0.09761235117912292, "loss/value_avg": 0.7088909149169922, "val/clipfrac_avg": 0.030641667544841766, "policy/entropy_avg": 0.16924285888671875, "val/ratio": 0.9882635474205017, "val/ratio_var": 7.515518518630415e-05, "val/num_eos_tokens": 454, "lr": 3.0865828381745515e-06, "episode": 12768, "epoch": 0.6448484848484849, "step": 798}, {"eps": 0, "objective/kl": 25.74894142150879, "objective/entropy": -101.5077133178711, "objective/non_score_reward": -1.2874470949172974, "objective/rlhf_reward": 13.735990524291992, "objective/scores": 15.0, "policy/approxkl_avg": 0.0007512210868299007, "policy/clipfrac_avg": 0.011982694268226624, "loss/policy_avg": 0.14146611094474792, "loss/value_avg": 0.7321012616157532, "val/clipfrac_avg": 0.012677992694079876, "policy/entropy_avg": 0.15892601013183594, "val/ratio": 0.9910134077072144, "val/ratio_var": 6.09022390563041e-05, "val/num_eos_tokens": 12, "lr": 3.074249318355046e-06, "episode": 12784, "epoch": 0.6456565656565657, "step": 799}, {"eps": 0, "objective/kl": 34.023399353027344, "objective/entropy": -68.88214874267578, "objective/non_score_reward": -1.7011699676513672, "objective/rlhf_reward": 12.494142532348633, "objective/scores": 14.1875, "policy/approxkl_avg": 0.0010239251423627138, "policy/clipfrac_avg": 0.010894561186432838, "loss/policy_avg": -0.042851828038692474, "loss/value_avg": 0.3452759385108948, "val/clipfrac_avg": 0.026722928509116173, "policy/entropy_avg": 0.17738628387451172, "val/ratio": 0.9888595342636108, "val/ratio_var": 6.888628558954224e-05, "val/num_eos_tokens": 67, "lr": 3.0619295416417856e-06, "episode": 12800, "epoch": 0.6464646464646465, "step": 800}, {"eps": 0, "objective/kl": 32.48891067504883, "objective/entropy": -89.62500762939453, "objective/non_score_reward": -1.6244455575942993, "objective/rlhf_reward": 14.227116584777832, "objective/scores": 15.875, "policy/approxkl_avg": 0.0009385538287460804, "policy/clipfrac_avg": 0.004920909181237221, "loss/policy_avg": -0.05754968896508217, "loss/value_avg": 0.402332067489624, "val/clipfrac_avg": 0.03176930546760559, "policy/entropy_avg": 0.17440450191497803, "val/ratio": 0.9880223274230957, "val/ratio_var": 5.388163845054805e-05, "val/num_eos_tokens": 9, "lr": 3.049623595954766e-06, "episode": 12816, "epoch": 0.6472727272727272, "step": 801}, {"eps": 0, "objective/kl": 40.699127197265625, "objective/entropy": -89.31193542480469, "objective/non_score_reward": -2.034956455230713, "objective/rlhf_reward": 12.160355567932129, "objective/scores": 14.1875, "policy/approxkl_avg": 0.0008679171442054212, "policy/clipfrac_avg": 0.004822667222470045, "loss/policy_avg": 0.2483101636171341, "loss/value_avg": 0.23584425449371338, "val/clipfrac_avg": 0.05813268944621086, "policy/entropy_avg": 0.1660480499267578, "val/ratio": 0.9890165328979492, "val/ratio_var": 5.875104761798866e-05, "val/num_eos_tokens": 149, "lr": 3.0373315691152856e-06, "episode": 12832, "epoch": 0.648080808080808, "step": 802}, {"eps": 0, "objective/kl": 38.51166915893555, "objective/entropy": -148.4324188232422, "objective/non_score_reward": -1.9255836009979248, "objective/rlhf_reward": 11.918167114257812, "objective/scores": 13.875, "policy/approxkl_avg": 0.0007962615345604718, "policy/clipfrac_avg": 0.0040261996909976006, "loss/policy_avg": -0.10593897104263306, "loss/value_avg": 0.5456434488296509, "val/clipfrac_avg": 0.04293014109134674, "policy/entropy_avg": 0.16094493865966797, "val/ratio": 0.9893560409545898, "val/ratio_var": 6.857667176518589e-05, "val/num_eos_tokens": 331, "lr": 3.0250535488453077e-06, "episode": 12848, "epoch": 0.6488888888888888, "step": 803}, {"eps": 0, "objective/kl": 40.921504974365234, "objective/entropy": -115.89801788330078, "objective/non_score_reward": -2.0460751056671143, "objective/rlhf_reward": 12.867987632751465, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0008664772612974048, "policy/clipfrac_avg": 0.00966357346624136, "loss/policy_avg": 0.06602153182029724, "loss/value_avg": 0.2823004722595215, "val/clipfrac_avg": 0.03888742998242378, "policy/entropy_avg": 0.151214599609375, "val/ratio": 0.9900601506233215, "val/ratio_var": 5.00909227412194e-05, "val/num_eos_tokens": 224, "lr": 3.0127896227668367e-06, "episode": 12864, "epoch": 0.6496969696969697, "step": 804}, {"eps": 0, "objective/kl": 35.056907653808594, "objective/entropy": -77.75199890136719, "objective/non_score_reward": -1.7528454065322876, "objective/rlhf_reward": 14.00496768951416, "objective/scores": 15.75, "policy/approxkl_avg": 0.001023960066959262, "policy/clipfrac_avg": 0.004629913717508316, "loss/policy_avg": -0.39892637729644775, "loss/value_avg": 0.4240272641181946, "val/clipfrac_avg": 0.07910472899675369, "policy/entropy_avg": 0.20476627349853516, "val/ratio": 0.9861233830451965, "val/ratio_var": 8.373144373763353e-05, "val/num_eos_tokens": 219, "lr": 3.000539878401296e-06, "episode": 12880, "epoch": 0.6505050505050505, "step": 805}, {"eps": 0, "objective/kl": 40.88092803955078, "objective/entropy": -85.76292419433594, "objective/non_score_reward": -2.044046401977539, "objective/rlhf_reward": 14.174703598022461, "objective/scores": 16.25, "policy/approxkl_avg": 0.0010081828804686666, "policy/clipfrac_avg": 0.0041284579783678055, "loss/policy_avg": -0.23323668539524078, "loss/value_avg": 0.4607469439506531, "val/clipfrac_avg": 0.16490784287452698, "policy/entropy_avg": 0.1921253204345703, "val/ratio": 0.9875838756561279, "val/ratio_var": 6.450649379985407e-05, "val/num_eos_tokens": 249, "lr": 2.9883044031688995e-06, "episode": 12896, "epoch": 0.6513131313131313, "step": 806}, {"eps": 0, "objective/kl": 33.47719955444336, "objective/entropy": -62.67930221557617, "objective/non_score_reward": -1.6738600730895996, "objective/rlhf_reward": 12.771451950073242, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0011091704946011305, "policy/clipfrac_avg": 0.00747725460678339, "loss/policy_avg": -0.11845395714044571, "loss/value_avg": 0.4592535197734833, "val/clipfrac_avg": 0.033114466816186905, "policy/entropy_avg": 0.1983814239501953, "val/ratio": 0.9877710938453674, "val/ratio_var": 6.532426050398499e-05, "val/num_eos_tokens": 17, "lr": 2.976083284388031e-06, "episode": 12912, "epoch": 0.6521212121212121, "step": 807}, {"eps": 0, "objective/kl": 34.25027847290039, "objective/entropy": -88.9464111328125, "objective/non_score_reward": -1.7125139236450195, "objective/rlhf_reward": 12.56873607635498, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009448326891288161, "policy/clipfrac_avg": 0.006387755274772644, "loss/policy_avg": 0.0469900444149971, "loss/value_avg": 0.26954370737075806, "val/clipfrac_avg": 0.018177732825279236, "policy/entropy_avg": 0.18566322326660156, "val/ratio": 0.9882063269615173, "val/ratio_var": 5.754105222877115e-05, "val/num_eos_tokens": 761, "lr": 2.9638766092746207e-06, "episode": 12928, "epoch": 0.6529292929292929, "step": 808}, {"eps": 0, "objective/kl": 40.86418151855469, "objective/entropy": -93.69944763183594, "objective/non_score_reward": -2.0432090759277344, "objective/rlhf_reward": 13.542728424072266, "objective/scores": 15.5625, "policy/approxkl_avg": 0.000799038156401366, "policy/clipfrac_avg": 0.006408605724573135, "loss/policy_avg": 0.00823296420276165, "loss/value_avg": 0.44794195890426636, "val/clipfrac_avg": 0.08976449072360992, "policy/entropy_avg": 0.14969253540039062, "val/ratio": 0.989780068397522, "val/ratio_var": 6.0727201343979686e-05, "val/num_eos_tokens": 215, "lr": 2.9516844649415165e-06, "episode": 12944, "epoch": 0.6537373737373737, "step": 809}, {"eps": 0, "objective/kl": 35.73942184448242, "objective/entropy": -41.68758773803711, "objective/non_score_reward": -1.7869713306427002, "objective/rlhf_reward": 15.314591407775879, "objective/scores": 17.125, "policy/approxkl_avg": 0.0008846010314300656, "policy/clipfrac_avg": 0.00692939106374979, "loss/policy_avg": 0.024010583758354187, "loss/value_avg": 0.3378852605819702, "val/clipfrac_avg": 0.0032868448179215193, "policy/entropy_avg": 0.17317676544189453, "val/ratio": 0.9892865419387817, "val/ratio_var": 4.3601932702586055e-05, "val/num_eos_tokens": 327, "lr": 2.9395069383978725e-06, "episode": 12960, "epoch": 0.6545454545454545, "step": 810}, {"eps": 0, "objective/kl": 36.01569366455078, "objective/entropy": -45.97090148925781, "objective/non_score_reward": -1.8007844686508179, "objective/rlhf_reward": 14.23046588897705, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009762131376191974, "policy/clipfrac_avg": 0.004803863354027271, "loss/policy_avg": 0.1456170380115509, "loss/value_avg": 0.5115708112716675, "val/clipfrac_avg": 0.07713419198989868, "policy/entropy_avg": 0.18830299377441406, "val/ratio": 0.9878898859024048, "val/ratio_var": 4.7962184908101335e-05, "val/num_eos_tokens": 267, "lr": 2.9273441165485227e-06, "episode": 12976, "epoch": 0.6553535353535354, "step": 811}, {"eps": 0, "objective/kl": 36.481346130371094, "objective/entropy": -66.14154052734375, "objective/non_score_reward": -1.8240673542022705, "objective/rlhf_reward": 14.175932884216309, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009823704604059458, "policy/clipfrac_avg": 0.0034461808390915394, "loss/policy_avg": 0.1002146378159523, "loss/value_avg": 0.558029294013977, "val/clipfrac_avg": 0.10748951137065887, "policy/entropy_avg": 0.1955432891845703, "val/ratio": 0.986059308052063, "val/ratio_var": 5.328569022822194e-05, "val/num_eos_tokens": 292, "lr": 2.9151960861933616e-06, "episode": 12992, "epoch": 0.6561616161616162, "step": 812}, {"eps": 0, "objective/kl": 36.021034240722656, "objective/entropy": -110.846923828125, "objective/non_score_reward": -1.8010516166687012, "objective/rlhf_reward": 16.33176040649414, "objective/scores": 18.125, "policy/approxkl_avg": 0.0008035163627937436, "policy/clipfrac_avg": 0.005940265022218227, "loss/policy_avg": -0.10509809851646423, "loss/value_avg": 0.46006062626838684, "val/clipfrac_avg": 0.07604548335075378, "policy/entropy_avg": 0.15110206604003906, "val/ratio": 0.9898133277893066, "val/ratio_var": 6.0903592384420335e-05, "val/num_eos_tokens": 523, "lr": 2.9030629340267165e-06, "episode": 13008, "epoch": 0.656969696969697, "step": 813}, {"eps": 0, "objective/kl": 34.03726577758789, "objective/entropy": -40.75424575805664, "objective/non_score_reward": -1.701863408088684, "objective/rlhf_reward": 13.091104507446289, "objective/scores": 14.8125, "policy/approxkl_avg": 0.000980376498773694, "policy/clipfrac_avg": 0.008426186628639698, "loss/policy_avg": -0.11711955070495605, "loss/value_avg": 1.2142528295516968, "val/clipfrac_avg": 0.02976190485060215, "policy/entropy_avg": 0.20235157012939453, "val/ratio": 0.9871314764022827, "val/ratio_var": 6.27562912995927e-05, "val/num_eos_tokens": 113, "lr": 2.890944746636748e-06, "episode": 13024, "epoch": 0.6577777777777778, "step": 814}, {"eps": 0, "objective/kl": 42.53788757324219, "objective/entropy": -141.41366577148438, "objective/non_score_reward": -2.126894474029541, "objective/rlhf_reward": 13.537168502807617, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008768361294642091, "policy/clipfrac_avg": 0.006732264067977667, "loss/policy_avg": -0.18143659830093384, "loss/value_avg": 0.571576714515686, "val/clipfrac_avg": 0.03276197984814644, "policy/entropy_avg": 0.17724037170410156, "val/ratio": 0.9885480403900146, "val/ratio_var": 8.229191007558256e-05, "val/num_eos_tokens": 221, "lr": 2.8788416105048124e-06, "episode": 13040, "epoch": 0.6585858585858586, "step": 815}, {"eps": 0, "objective/kl": 35.6143684387207, "objective/entropy": -61.10264587402344, "objective/non_score_reward": -1.7807183265686035, "objective/rlhf_reward": 14.391157150268555, "objective/scores": 16.125, "policy/approxkl_avg": 0.0009194887243211269, "policy/clipfrac_avg": 0.018624408170580864, "loss/policy_avg": 0.3017633855342865, "loss/value_avg": 0.32543864846229553, "val/clipfrac_avg": 0.0073128631338477135, "policy/entropy_avg": 0.14903783798217773, "val/ratio": 0.989689290523529, "val/ratio_var": 5.833535033161752e-05, "val/num_eos_tokens": 112, "lr": 2.8667536120048616e-06, "episode": 13056, "epoch": 0.6593939393939394, "step": 816}, {"eps": 0, "objective/kl": 34.15174102783203, "objective/entropy": -117.86632537841797, "objective/non_score_reward": -1.7075870037078857, "objective/rlhf_reward": 14.511162757873535, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009505614871159196, "policy/clipfrac_avg": 0.008451011963188648, "loss/policy_avg": 0.06899265199899673, "loss/value_avg": 0.6989455223083496, "val/clipfrac_avg": 0.000589622650295496, "policy/entropy_avg": 0.1699972152709961, "val/ratio": 0.9876314401626587, "val/ratio_var": 9.73327478277497e-05, "val/num_eos_tokens": 435, "lr": 2.854680837402804e-06, "episode": 13072, "epoch": 0.6602020202020202, "step": 817}, {"eps": 0, "objective/kl": 40.512550354003906, "objective/entropy": -52.28473663330078, "objective/non_score_reward": -2.025627613067627, "objective/rlhf_reward": 14.458747863769531, "objective/scores": 16.5, "policy/approxkl_avg": 0.0008845083648338914, "policy/clipfrac_avg": 0.005094105843454599, "loss/policy_avg": -0.05428570508956909, "loss/value_avg": 0.2928343415260315, "val/clipfrac_avg": 0.09271319210529327, "policy/entropy_avg": 0.16685867309570312, "val/ratio": 0.9885064959526062, "val/ratio_var": 4.853576319874264e-05, "val/num_eos_tokens": 32, "lr": 2.8426233728559172e-06, "episode": 13088, "epoch": 0.661010101010101, "step": 818}, {"eps": 0, "objective/kl": 42.012672424316406, "objective/entropy": -196.3238983154297, "objective/non_score_reward": -2.1006336212158203, "objective/rlhf_reward": 12.51655387878418, "objective/scores": 14.625, "policy/approxkl_avg": 0.0007904545636847615, "policy/clipfrac_avg": 0.006419865880161524, "loss/policy_avg": 0.12085426598787308, "loss/value_avg": 0.5670195817947388, "val/clipfrac_avg": 0.05388574302196503, "policy/entropy_avg": 0.1437664031982422, "val/ratio": 0.9899193644523621, "val/ratio_var": 4.292546145734377e-05, "val/num_eos_tokens": 496, "lr": 2.83058130441221e-06, "episode": 13104, "epoch": 0.6618181818181819, "step": 819}, {"eps": 0, "objective/kl": 38.73785400390625, "objective/entropy": -58.513553619384766, "objective/non_score_reward": -1.9368927478790283, "objective/rlhf_reward": 12.883419036865234, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0010288474150002003, "policy/clipfrac_avg": 0.021498143672943115, "loss/policy_avg": 0.6058844327926636, "loss/value_avg": 0.7294787168502808, "val/clipfrac_avg": 0.0723545029759407, "policy/entropy_avg": 0.20896148681640625, "val/ratio": 0.9887969493865967, "val/ratio_var": 4.8135509132407606e-05, "val/num_eos_tokens": 134, "lr": 2.8185547180098195e-06, "episode": 13120, "epoch": 0.6626262626262627, "step": 820}, {"eps": 0, "objective/kl": 33.36050033569336, "objective/entropy": -38.43750762939453, "objective/non_score_reward": -1.6680251359939575, "objective/rlhf_reward": 13.542912483215332, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0010698122205212712, "policy/clipfrac_avg": 0.004931025672703981, "loss/policy_avg": -0.21932660043239594, "loss/value_avg": 0.45023149251937866, "val/clipfrac_avg": 0.00852490495890379, "policy/entropy_avg": 0.19058465957641602, "val/ratio": 0.9864101409912109, "val/ratio_var": 5.521172352018766e-05, "val/num_eos_tokens": 190, "lr": 2.8065436994763963e-06, "episode": 13136, "epoch": 0.6634343434343435, "step": 821}, {"eps": 0, "objective/kl": 44.57916259765625, "objective/entropy": 13.10763931274414, "objective/non_score_reward": -2.2289581298828125, "objective/rlhf_reward": 13.364792823791504, "objective/scores": 15.625, "policy/approxkl_avg": 0.0014038155786693096, "policy/clipfrac_avg": 0.006509849801659584, "loss/policy_avg": -0.01703064516186714, "loss/value_avg": 0.2902008295059204, "val/clipfrac_avg": 0.03128276392817497, "policy/entropy_avg": 0.26366424560546875, "val/ratio": 0.9824488162994385, "val/ratio_var": 4.7705398174002767e-05, "val/num_eos_tokens": 35, "lr": 2.794548334528486e-06, "episode": 13152, "epoch": 0.6642424242424242, "step": 822}, {"eps": 0, "objective/kl": 42.75294494628906, "objective/entropy": -94.84840393066406, "objective/non_score_reward": -2.1376476287841797, "objective/rlhf_reward": 12.26860237121582, "objective/scores": 14.375, "policy/approxkl_avg": 0.0010491260327398777, "policy/clipfrac_avg": 0.004985281266272068, "loss/policy_avg": -0.02819274365901947, "loss/value_avg": 0.2861873507499695, "val/clipfrac_avg": 0.00754884909838438, "policy/entropy_avg": 0.2000885009765625, "val/ratio": 0.9866741895675659, "val/ratio_var": 4.918476770399138e-05, "val/num_eos_tokens": 289, "lr": 2.782568708770933e-06, "episode": 13168, "epoch": 0.665050505050505, "step": 823}, {"eps": 0, "objective/kl": 45.08838653564453, "objective/entropy": -104.7535400390625, "objective/non_score_reward": -2.2544193267822266, "objective/rlhf_reward": 12.214330673217773, "objective/scores": 14.5, "policy/approxkl_avg": 0.0009155337465927005, "policy/clipfrac_avg": 0.008831838145852089, "loss/policy_avg": -0.024818487465381622, "loss/value_avg": 0.3611629605293274, "val/clipfrac_avg": 0.0024258445482701063, "policy/entropy_avg": 0.16748523712158203, "val/ratio": 0.9890375137329102, "val/ratio_var": 5.131318175699562e-05, "val/num_eos_tokens": 163, "lr": 2.7706049076962483e-06, "episode": 13184, "epoch": 0.6658585858585858, "step": 824}, {"eps": 0, "objective/kl": 30.59856605529785, "objective/entropy": -74.55836486816406, "objective/non_score_reward": -1.52992844581604, "objective/rlhf_reward": 15.329446792602539, "objective/scores": 16.875, "policy/approxkl_avg": 0.0008330290438607335, "policy/clipfrac_avg": 0.010464916005730629, "loss/policy_avg": -0.28224268555641174, "loss/value_avg": 0.8720104694366455, "val/clipfrac_avg": 0.08580771088600159, "policy/entropy_avg": 0.15100550651550293, "val/ratio": 0.9896235466003418, "val/ratio_var": 9.328931628260761e-05, "val/num_eos_tokens": 460, "lr": 2.7586570166840154e-06, "episode": 13200, "epoch": 0.6666666666666666, "step": 825}, {"eps": 0, "objective/kl": 41.760223388671875, "objective/entropy": -43.89616394042969, "objective/non_score_reward": -2.0880112648010254, "objective/rlhf_reward": 13.872926712036133, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0010317859705537558, "policy/clipfrac_avg": 0.0054151020012795925, "loss/policy_avg": -0.1715128868818283, "loss/value_avg": 0.4540337920188904, "val/clipfrac_avg": 0.03631855547428131, "policy/entropy_avg": 0.19500446319580078, "val/ratio": 0.9874616265296936, "val/ratio_var": 6.214805034687743e-05, "val/num_eos_tokens": 334, "lr": 2.746725121000273e-06, "episode": 13216, "epoch": 0.6674747474747474, "step": 826}, {"eps": 0, "objective/kl": 34.05264663696289, "objective/entropy": -13.071683883666992, "objective/non_score_reward": -1.7026324272155762, "objective/rlhf_reward": 14.656742095947266, "objective/scores": 16.375, "policy/approxkl_avg": 0.0012533586705103517, "policy/clipfrac_avg": 0.00517519935965538, "loss/policy_avg": -0.077702596783638, "loss/value_avg": 0.6576616168022156, "val/clipfrac_avg": 0.09026823937892914, "policy/entropy_avg": 0.21735763549804688, "val/ratio": 0.9863014221191406, "val/ratio_var": 0.00010488701082067564, "val/num_eos_tokens": 468, "lr": 2.734809305796915e-06, "episode": 13232, "epoch": 0.6682828282828283, "step": 827}, {"eps": 0, "objective/kl": 37.0118522644043, "objective/entropy": -132.30560302734375, "objective/non_score_reward": -1.8505926132202148, "objective/rlhf_reward": 15.282218933105469, "objective/scores": 17.125, "policy/approxkl_avg": 0.0008336640894412994, "policy/clipfrac_avg": 0.0030343476682901382, "loss/policy_avg": -0.11604561656713486, "loss/value_avg": 0.18491807579994202, "val/clipfrac_avg": 0.07020965218544006, "policy/entropy_avg": 0.16221046447753906, "val/ratio": 0.9894965291023254, "val/ratio_var": 4.505051037995145e-05, "val/num_eos_tokens": 41, "lr": 2.7229096561110703e-06, "episode": 13248, "epoch": 0.6690909090909091, "step": 828}, {"eps": 0, "objective/kl": 42.80398941040039, "objective/entropy": -83.5711441040039, "objective/non_score_reward": -2.1401994228363037, "objective/rlhf_reward": 13.555112838745117, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008490884792990983, "policy/clipfrac_avg": 0.024211544543504715, "loss/policy_avg": -0.23082350194454193, "loss/value_avg": 0.9338909387588501, "val/clipfrac_avg": 0.01703277789056301, "policy/entropy_avg": 0.16208171844482422, "val/ratio": 0.9889751672744751, "val/ratio_var": 6.549843965331092e-05, "val/num_eos_tokens": 407, "lr": 2.7110262568645063e-06, "episode": 13264, "epoch": 0.6698989898989899, "step": 829}, {"eps": 0, "objective/kl": 30.28129005432129, "objective/entropy": -91.52725219726562, "objective/non_score_reward": -1.5140645503997803, "objective/rlhf_reward": 13.71249771118164, "objective/scores": 15.25, "policy/approxkl_avg": 0.0006465939804911613, "policy/clipfrac_avg": 0.003674568608403206, "loss/policy_avg": -0.1790890097618103, "loss/value_avg": 0.9402360320091248, "val/clipfrac_avg": 0.05747377499938011, "policy/entropy_avg": 0.14063799381256104, "val/ratio": 0.9919025897979736, "val/ratio_var": 6.961268809391186e-05, "val/num_eos_tokens": 250, "lr": 2.699159192863018e-06, "episode": 13280, "epoch": 0.6707070707070707, "step": 830}, {"eps": 0, "objective/kl": 44.12242889404297, "objective/entropy": -54.39606475830078, "objective/non_score_reward": -2.2061214447021484, "objective/rlhf_reward": 14.004816055297852, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009418983245268464, "policy/clipfrac_avg": 0.009523827582597733, "loss/policy_avg": 0.3400639295578003, "loss/value_avg": 1.9688762426376343, "val/clipfrac_avg": 0.045346200466156006, "policy/entropy_avg": 0.18965661525726318, "val/ratio": 0.9880996942520142, "val/ratio_var": 6.565819057868794e-05, "val/num_eos_tokens": 245, "lr": 2.687308548795825e-06, "episode": 13296, "epoch": 0.6715151515151515, "step": 831}, {"eps": 0, "objective/kl": 38.93806838989258, "objective/entropy": -66.36386108398438, "objective/non_score_reward": -1.9469034671783447, "objective/rlhf_reward": 14.076534271240234, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008638559957034886, "policy/clipfrac_avg": 0.005322727840393782, "loss/policy_avg": 0.11609108746051788, "loss/value_avg": 0.5635161399841309, "val/clipfrac_avg": 0.04381619766354561, "policy/entropy_avg": 0.15152549743652344, "val/ratio": 0.9889363050460815, "val/ratio_var": 3.002676021424122e-05, "val/num_eos_tokens": 20, "lr": 2.675474409234965e-06, "episode": 13312, "epoch": 0.6723232323232323, "step": 832}, {"eps": 0, "objective/kl": 47.16798400878906, "objective/entropy": -111.39958953857422, "objective/non_score_reward": -2.3583993911743164, "objective/rlhf_reward": 11.7197265625, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0009453936363570392, "policy/clipfrac_avg": 0.004603059496730566, "loss/policy_avg": -0.03308400511741638, "loss/value_avg": 0.3210276961326599, "val/clipfrac_avg": 0.062541663646698, "policy/entropy_avg": 0.18264532089233398, "val/ratio": 0.9879401922225952, "val/ratio_var": 6.487595965154469e-05, "val/num_eos_tokens": 763, "lr": 2.66365685863469e-06, "episode": 13328, "epoch": 0.6731313131313131, "step": 833}, {"eps": 0, "objective/kl": 37.68748474121094, "objective/entropy": -69.66492462158203, "objective/non_score_reward": -1.8843741416931152, "objective/rlhf_reward": 12.764063835144043, "objective/scores": 14.625, "policy/approxkl_avg": 0.0010055399034172297, "policy/clipfrac_avg": 0.004349449649453163, "loss/policy_avg": 0.051825061440467834, "loss/value_avg": 0.8246160745620728, "val/clipfrac_avg": 0.06374436616897583, "policy/entropy_avg": 0.18214893341064453, "val/ratio": 0.9867125749588013, "val/ratio_var": 9.083253826247528e-05, "val/num_eos_tokens": 38, "lr": 2.651855981330872e-06, "episode": 13344, "epoch": 0.673939393939394, "step": 834}, {"eps": 0, "objective/kl": 39.288482666015625, "objective/entropy": -108.79554748535156, "objective/non_score_reward": -1.9644241333007812, "objective/rlhf_reward": 14.504325866699219, "objective/scores": 16.5, "policy/approxkl_avg": 0.0010659170802682638, "policy/clipfrac_avg": 0.005707671865820885, "loss/policy_avg": -0.043558381497859955, "loss/value_avg": 0.6093795895576477, "val/clipfrac_avg": 0.013371335342526436, "policy/entropy_avg": 0.1986398696899414, "val/ratio": 0.9869523048400879, "val/ratio_var": 8.708865061635152e-05, "val/num_eos_tokens": 647, "lr": 2.6400718615403852e-06, "episode": 13360, "epoch": 0.6747474747474748, "step": 835}, {"eps": 0, "objective/kl": 42.06044387817383, "objective/entropy": -59.1908073425293, "objective/non_score_reward": -2.1030220985412598, "objective/rlhf_reward": 13.842290878295898, "objective/scores": 15.9375, "policy/approxkl_avg": 0.001016529044136405, "policy/clipfrac_avg": 0.007354079745709896, "loss/policy_avg": 0.09040576219558716, "loss/value_avg": 0.4517725706100464, "val/clipfrac_avg": 0.030187882483005524, "policy/entropy_avg": 0.20728302001953125, "val/ratio": 0.9865027666091919, "val/ratio_var": 5.405073534348048e-05, "val/num_eos_tokens": 477, "lr": 2.62830458336052e-06, "episode": 13376, "epoch": 0.6755555555555556, "step": 836}, {"eps": 0, "objective/kl": 43.636474609375, "objective/entropy": -99.01640319824219, "objective/non_score_reward": -2.18182373046875, "objective/rlhf_reward": 13.08380126953125, "objective/scores": 15.25, "policy/approxkl_avg": 0.0009176525054499507, "policy/clipfrac_avg": 0.006078221835196018, "loss/policy_avg": -0.21371892094612122, "loss/value_avg": 0.26992350816726685, "val/clipfrac_avg": 0.025651777163147926, "policy/entropy_avg": 0.1714763641357422, "val/ratio": 0.988213300704956, "val/ratio_var": 6.253684114199132e-05, "val/num_eos_tokens": 453, "lr": 2.6165542307683744e-06, "episode": 13392, "epoch": 0.6763636363636364, "step": 837}, {"eps": 0, "objective/kl": 40.305843353271484, "objective/entropy": -62.21269989013672, "objective/non_score_reward": -2.015292167663574, "objective/rlhf_reward": 15.75814437866211, "objective/scores": 17.75, "policy/approxkl_avg": 0.0009111049002967775, "policy/clipfrac_avg": 0.005017759278416634, "loss/policy_avg": 0.08225658535957336, "loss/value_avg": 0.4106031656265259, "val/clipfrac_avg": 0.0028512999415397644, "policy/entropy_avg": 0.18220901489257812, "val/ratio": 0.9878619909286499, "val/ratio_var": 3.237010969314724e-05, "val/num_eos_tokens": 268, "lr": 2.604820887620254e-06, "episode": 13408, "epoch": 0.6771717171717172, "step": 838}, {"eps": 0, "objective/kl": 35.24858093261719, "objective/entropy": -38.304931640625, "objective/non_score_reward": -1.7624289989471436, "objective/rlhf_reward": 12.284445762634277, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0008708706009201705, "policy/clipfrac_avg": 0.005640837829560041, "loss/policy_avg": 0.01355072483420372, "loss/value_avg": 0.4876829981803894, "val/clipfrac_avg": 0.029899097979068756, "policy/entropy_avg": 0.16802191734313965, "val/ratio": 0.989201009273529, "val/ratio_var": 5.1689257816178724e-05, "val/num_eos_tokens": 258, "lr": 2.5931046376510875e-06, "episode": 13424, "epoch": 0.677979797979798, "step": 839}, {"eps": 0, "objective/kl": 36.323509216308594, "objective/entropy": -63.923397064208984, "objective/non_score_reward": -1.8161752223968506, "objective/rlhf_reward": 12.97288703918457, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0009589061955921352, "policy/clipfrac_avg": 0.006033012643456459, "loss/policy_avg": 0.1623387336730957, "loss/value_avg": 0.3584904074668884, "val/clipfrac_avg": 0.11569125950336456, "policy/entropy_avg": 0.17229270935058594, "val/ratio": 0.988390326499939, "val/ratio_var": 7.233954966068268e-05, "val/num_eos_tokens": 159, "lr": 2.5814055644738013e-06, "episode": 13440, "epoch": 0.6787878787878788, "step": 840}, {"eps": 0, "objective/kl": 34.90948486328125, "objective/entropy": -159.0351104736328, "objective/non_score_reward": -1.7454744577407837, "objective/rlhf_reward": 13.145150184631348, "objective/scores": 14.875, "policy/approxkl_avg": 0.0007017417810857296, "policy/clipfrac_avg": 0.005396063905209303, "loss/policy_avg": -0.29571330547332764, "loss/value_avg": 0.5451829433441162, "val/clipfrac_avg": 0.019056372344493866, "policy/entropy_avg": 0.12675732374191284, "val/ratio": 0.9914215207099915, "val/ratio_var": 8.740482735447586e-05, "val/num_eos_tokens": 508, "lr": 2.5697237515787564e-06, "episode": 13456, "epoch": 0.6795959595959596, "step": 841}, {"eps": 0, "objective/kl": 36.281490325927734, "objective/entropy": -83.5593490600586, "objective/non_score_reward": -1.8140745162963867, "objective/rlhf_reward": 14.18592643737793, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009474118705838919, "policy/clipfrac_avg": 0.004876702558249235, "loss/policy_avg": -0.33980700373649597, "loss/value_avg": 0.5796597003936768, "val/clipfrac_avg": 0.03151373565196991, "policy/entropy_avg": 0.14034080505371094, "val/ratio": 0.9901444911956787, "val/ratio_var": 6.344315625028685e-05, "val/num_eos_tokens": 208, "lr": 2.5580592823331237e-06, "episode": 13472, "epoch": 0.6804040404040405, "step": 842}, {"eps": 0, "objective/kl": 46.23011779785156, "objective/entropy": -74.54005432128906, "objective/non_score_reward": -2.3115057945251465, "objective/rlhf_reward": 12.993181228637695, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0009884861065074801, "policy/clipfrac_avg": 0.006667924579232931, "loss/policy_avg": 0.11847849935293198, "loss/value_avg": 0.3053319454193115, "val/clipfrac_avg": 0.00010080645006382838, "policy/entropy_avg": 0.19658470153808594, "val/ratio": 0.9878820180892944, "val/ratio_var": 4.166604412603192e-05, "val/num_eos_tokens": 52, "lr": 2.5464122399803126e-06, "episode": 13488, "epoch": 0.6812121212121212, "step": 843}, {"eps": 0, "objective/kl": 36.799835205078125, "objective/entropy": -55.136016845703125, "objective/non_score_reward": -1.8399916887283325, "objective/rlhf_reward": 12.995945930480957, "objective/scores": 14.8125, "policy/approxkl_avg": 0.0010256943060085177, "policy/clipfrac_avg": 0.009538116864860058, "loss/policy_avg": 0.07477365434169769, "loss/value_avg": 0.717211902141571, "val/clipfrac_avg": 0.0769520252943039, "policy/entropy_avg": 0.19795989990234375, "val/ratio": 0.9861817955970764, "val/ratio_var": 7.031750283204019e-05, "val/num_eos_tokens": 573, "lr": 2.5347827076393505e-06, "episode": 13504, "epoch": 0.682020202020202, "step": 844}, {"eps": 0, "objective/kl": 41.09418487548828, "objective/entropy": -164.994873046875, "objective/non_score_reward": -2.0547094345092773, "objective/rlhf_reward": 12.367165565490723, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0006862258305773139, "policy/clipfrac_avg": 0.005633420776575804, "loss/policy_avg": -0.010380454361438751, "loss/value_avg": 0.8978620767593384, "val/clipfrac_avg": 0.10761284828186035, "policy/entropy_avg": 0.136183500289917, "val/ratio": 0.9907996654510498, "val/ratio_var": 4.169603198533878e-05, "val/num_eos_tokens": 289, "lr": 2.5231707683043204e-06, "episode": 13520, "epoch": 0.6828282828282828, "step": 845}, {"eps": 0, "objective/kl": 37.73551940917969, "objective/entropy": -119.26876068115234, "objective/non_score_reward": -1.8867759704589844, "objective/rlhf_reward": 13.769474029541016, "objective/scores": 15.625, "policy/approxkl_avg": 0.0007093651220202446, "policy/clipfrac_avg": 0.028275154531002045, "loss/policy_avg": -0.0011942088603973389, "loss/value_avg": 0.7805696129798889, "val/clipfrac_avg": 0.08967090398073196, "policy/entropy_avg": 0.1421506404876709, "val/ratio": 0.9911968111991882, "val/ratio_var": 3.498927253531292e-05, "val/num_eos_tokens": 760, "lr": 2.5115765048437445e-06, "episode": 13536, "epoch": 0.6836363636363636, "step": 846}, {"eps": 0, "objective/kl": 34.76280975341797, "objective/entropy": -89.728271484375, "objective/non_score_reward": -1.7381404638290405, "objective/rlhf_reward": 13.925922393798828, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0009024562314152718, "policy/clipfrac_avg": 0.007501127198338509, "loss/policy_avg": -0.007019490003585815, "loss/value_avg": 0.8198627233505249, "val/clipfrac_avg": 0.028372831642627716, "policy/entropy_avg": 0.17042064666748047, "val/ratio": 0.9899512529373169, "val/ratio_var": 7.211479532998055e-05, "val/num_eos_tokens": 17, "lr": 2.5000000000000015e-06, "episode": 13552, "epoch": 0.6844444444444444, "step": 847}, {"eps": 0, "objective/kl": 46.20475769042969, "objective/entropy": -184.20425415039062, "objective/non_score_reward": -2.3102383613586426, "objective/rlhf_reward": 11.728824615478516, "objective/scores": 14.0625, "policy/approxkl_avg": 0.0007638686802238226, "policy/clipfrac_avg": 0.005230149254202843, "loss/policy_avg": 0.015100933611392975, "loss/value_avg": 0.25284868478775024, "val/clipfrac_avg": 0.01555095985531807, "policy/entropy_avg": 0.15384864807128906, "val/ratio": 0.9898365139961243, "val/ratio_var": 5.412843165686354e-05, "val/num_eos_tokens": 404, "lr": 2.488441336388737e-06, "episode": 13568, "epoch": 0.6852525252525252, "step": 848}, {"eps": 0, "objective/kl": 37.31745147705078, "objective/entropy": -70.96700286865234, "objective/non_score_reward": -1.8658727407455444, "objective/rlhf_reward": 13.665377616882324, "objective/scores": 15.5, "policy/approxkl_avg": 0.000885509536601603, "policy/clipfrac_avg": 0.006637534126639366, "loss/policy_avg": -0.13074953854084015, "loss/value_avg": 0.5254482626914978, "val/clipfrac_avg": 0.050514861941337585, "policy/entropy_avg": 0.17204856872558594, "val/ratio": 0.9882212281227112, "val/ratio_var": 5.447021248983219e-05, "val/num_eos_tokens": 24, "lr": 2.4769005964982718e-06, "episode": 13584, "epoch": 0.686060606060606, "step": 849}, {"eps": 0, "objective/kl": 48.39326477050781, "objective/entropy": -62.67339324951172, "objective/non_score_reward": -2.419663190841675, "objective/rlhf_reward": 13.564711570739746, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009431451326236129, "policy/clipfrac_avg": 0.004864457994699478, "loss/policy_avg": 0.039310187101364136, "loss/value_avg": 0.18585552275180817, "val/clipfrac_avg": 0.01945144683122635, "policy/entropy_avg": 0.19204330444335938, "val/ratio": 0.9872653484344482, "val/ratio_var": 3.194732926203869e-05, "val/num_eos_tokens": 36, "lr": 2.465377862689019e-06, "episode": 13600, "epoch": 0.6868686868686869, "step": 850}, {"eps": 0, "objective/kl": 35.364593505859375, "objective/entropy": -76.03438568115234, "objective/non_score_reward": -1.7682294845581055, "objective/rlhf_reward": 15.075519561767578, "objective/scores": 16.875, "policy/approxkl_avg": 0.0008544789161533117, "policy/clipfrac_avg": 0.005153194069862366, "loss/policy_avg": 0.07975710183382034, "loss/value_avg": 0.478973388671875, "val/clipfrac_avg": 0.03764945641160011, "policy/entropy_avg": 0.16641998291015625, "val/ratio": 0.9884979724884033, "val/ratio_var": 4.7100515075726435e-05, "val/num_eos_tokens": 298, "lr": 2.4538732171928847e-06, "episode": 13616, "epoch": 0.6876767676767677, "step": 851}, {"eps": 0, "objective/kl": 35.15203857421875, "objective/entropy": -59.744590759277344, "objective/non_score_reward": -1.7576020956039429, "objective/rlhf_reward": 15.05489730834961, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009704260155558586, "policy/clipfrac_avg": 0.004688786808401346, "loss/policy_avg": -0.10211583226919174, "loss/value_avg": 0.7199238538742065, "val/clipfrac_avg": 0.17230291664600372, "policy/entropy_avg": 0.17905235290527344, "val/ratio": 0.9872374534606934, "val/ratio_var": 4.29930878453888e-05, "val/num_eos_tokens": 644, "lr": 2.4423867421126923e-06, "episode": 13632, "epoch": 0.6884848484848485, "step": 852}, {"eps": 0, "objective/kl": 41.87160110473633, "objective/entropy": -63.859352111816406, "objective/non_score_reward": -2.0935800075531006, "objective/rlhf_reward": 16.01579475402832, "objective/scores": 18.125, "policy/approxkl_avg": 0.0011341955978423357, "policy/clipfrac_avg": 0.005558031145483255, "loss/policy_avg": 0.0028443392366170883, "loss/value_avg": 0.3421381115913391, "val/clipfrac_avg": 0.07718860357999802, "policy/entropy_avg": 0.21439552307128906, "val/ratio": 0.9858847856521606, "val/ratio_var": 5.448433512356132e-05, "val/num_eos_tokens": 400, "lr": 2.4309185194215885e-06, "episode": 13648, "epoch": 0.6892929292929293, "step": 853}, {"eps": 0, "objective/kl": 39.14732360839844, "objective/entropy": -138.73922729492188, "objective/non_score_reward": -1.9573662281036377, "objective/rlhf_reward": 11.542634010314941, "objective/scores": 13.5, "policy/approxkl_avg": 0.0011413603788241744, "policy/clipfrac_avg": 0.008165586739778519, "loss/policy_avg": 0.11498575657606125, "loss/value_avg": 0.909265398979187, "val/clipfrac_avg": 0.04511754959821701, "policy/entropy_avg": 0.18193864822387695, "val/ratio": 0.9880124926567078, "val/ratio_var": 7.85917291068472e-05, "val/num_eos_tokens": 264, "lr": 2.4194686309624664e-06, "episode": 13664, "epoch": 0.6901010101010101, "step": 854}, {"eps": 0, "objective/kl": 39.87467956542969, "objective/entropy": -80.8604736328125, "objective/non_score_reward": -1.9937340021133423, "objective/rlhf_reward": 12.850015640258789, "objective/scores": 14.875, "policy/approxkl_avg": 0.000854466634336859, "policy/clipfrac_avg": 0.0031361770816147327, "loss/policy_avg": 0.4261026978492737, "loss/value_avg": 1.0351085662841797, "val/clipfrac_avg": 0.06461638957262039, "policy/entropy_avg": 0.1807117462158203, "val/ratio": 0.9892903566360474, "val/ratio_var": 6.158453470561653e-05, "val/num_eos_tokens": 76, "lr": 2.408037158447375e-06, "episode": 13680, "epoch": 0.6909090909090909, "step": 855}, {"eps": 0, "objective/kl": 44.59175109863281, "objective/entropy": -109.00687408447266, "objective/non_score_reward": -2.2295875549316406, "objective/rlhf_reward": 14.02041244506836, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008301680209115148, "policy/clipfrac_avg": 0.004295141436159611, "loss/policy_avg": 0.35515934228897095, "loss/value_avg": 0.6004530787467957, "val/clipfrac_avg": 0.0578775517642498, "policy/entropy_avg": 0.16374778747558594, "val/ratio": 0.9892109632492065, "val/ratio_var": 3.649172867881134e-05, "val/num_eos_tokens": 355, "lr": 2.3966241834569364e-06, "episode": 13696, "epoch": 0.6917171717171717, "step": 856}, {"eps": 0, "objective/kl": 33.233463287353516, "objective/entropy": -65.75347137451172, "objective/non_score_reward": -1.6616731882095337, "objective/rlhf_reward": 12.885202407836914, "objective/scores": 14.5625, "policy/approxkl_avg": 0.0008445363491773605, "policy/clipfrac_avg": 0.0028887931257486343, "loss/policy_avg": -0.07683120667934418, "loss/value_avg": 0.7507062554359436, "val/clipfrac_avg": 0.029584139585494995, "policy/entropy_avg": 0.1688997745513916, "val/ratio": 0.9893004894256592, "val/ratio_var": 4.4566000724444166e-05, "val/num_eos_tokens": 79, "lr": 2.385229787439768e-06, "episode": 13712, "epoch": 0.6925252525252525, "step": 857}, {"eps": 0, "objective/kl": 39.995941162109375, "objective/entropy": -137.21546936035156, "objective/non_score_reward": -1.9997971057891846, "objective/rlhf_reward": 12.875202178955078, "objective/scores": 14.875, "policy/approxkl_avg": 0.0009317261283285916, "policy/clipfrac_avg": 0.016368409618735313, "loss/policy_avg": -0.04102202132344246, "loss/value_avg": 0.295141339302063, "val/clipfrac_avg": 0.02603679709136486, "policy/entropy_avg": 0.18145465850830078, "val/ratio": 0.9879333972930908, "val/ratio_var": 7.448039104929194e-05, "val/num_eos_tokens": 133, "lr": 2.3738540517118953e-06, "episode": 13728, "epoch": 0.6933333333333334, "step": 858}, {"eps": 0, "objective/kl": 41.33527755737305, "objective/entropy": -126.39140319824219, "objective/non_score_reward": -2.0667638778686523, "objective/rlhf_reward": 15.823860168457031, "objective/scores": 17.875, "policy/approxkl_avg": 0.0009032993111759424, "policy/clipfrac_avg": 0.008262147195637226, "loss/policy_avg": -0.09325989335775375, "loss/value_avg": 0.4778023660182953, "val/clipfrac_avg": 0.006358179729431868, "policy/entropy_avg": 0.1809234619140625, "val/ratio": 0.9881711602210999, "val/ratio_var": 8.430143498117104e-05, "val/num_eos_tokens": 198, "lr": 2.3624970574561773e-06, "episode": 13744, "epoch": 0.6941414141414142, "step": 859}, {"eps": 0, "objective/kl": 36.61549377441406, "objective/entropy": -82.32437133789062, "objective/non_score_reward": -1.8307745456695557, "objective/rlhf_reward": 14.497350692749023, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009405683958902955, "policy/clipfrac_avg": 0.0031182412058115005, "loss/policy_avg": -0.1992878019809723, "loss/value_avg": 0.4450816214084625, "val/clipfrac_avg": 0.04030741751194, "policy/entropy_avg": 0.1852397918701172, "val/ratio": 0.9874944686889648, "val/ratio_var": 5.9595575294224545e-05, "val/num_eos_tokens": 674, "lr": 2.3511588857217228e-06, "episode": 13760, "epoch": 0.694949494949495, "step": 860}, {"eps": 0, "objective/kl": 32.36253356933594, "objective/entropy": -97.76516723632812, "objective/non_score_reward": -1.6181267499923706, "objective/rlhf_reward": 14.108436584472656, "objective/scores": 15.75, "policy/approxkl_avg": 0.0008775125024840236, "policy/clipfrac_avg": 0.006763250567018986, "loss/policy_avg": -0.0025226399302482605, "loss/value_avg": 0.6712653636932373, "val/clipfrac_avg": 0.01572282984852791, "policy/entropy_avg": 0.15996170043945312, "val/ratio": 0.9897935390472412, "val/ratio_var": 4.971764428773895e-05, "val/num_eos_tokens": 755, "lr": 2.339839617423318e-06, "episode": 13776, "epoch": 0.6957575757575758, "step": 861}, {"eps": 0, "objective/kl": 36.61561584472656, "objective/entropy": -61.261680603027344, "objective/non_score_reward": -1.8307808637619019, "objective/rlhf_reward": 12.762969970703125, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009761417168192565, "policy/clipfrac_avg": 0.0026826506946235895, "loss/policy_avg": 0.08292508125305176, "loss/value_avg": 0.5537270307540894, "val/clipfrac_avg": 0.044523607939481735, "policy/entropy_avg": 0.22123146057128906, "val/ratio": 0.9871484041213989, "val/ratio_var": 7.335093687288463e-05, "val/num_eos_tokens": 56, "lr": 2.32853933334084e-06, "episode": 13792, "epoch": 0.6965656565656566, "step": 862}, {"eps": 0, "objective/kl": 36.92033004760742, "objective/entropy": -76.75859069824219, "objective/non_score_reward": -1.8460166454315186, "objective/rlhf_reward": 14.872733116149902, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009935328271239996, "policy/clipfrac_avg": 0.006773949135094881, "loss/policy_avg": 0.36910420656204224, "loss/value_avg": 0.9228214025497437, "val/clipfrac_avg": 0.07717607915401459, "policy/entropy_avg": 0.1806344985961914, "val/ratio": 0.9876723289489746, "val/ratio_var": 6.279607623582706e-05, "val/num_eos_tokens": 514, "lr": 2.317258114118686e-06, "episode": 13808, "epoch": 0.6973737373737374, "step": 863}, {"eps": 0, "objective/kl": 45.520545959472656, "objective/entropy": -77.51583862304688, "objective/non_score_reward": -2.276027202606201, "objective/rlhf_reward": 14.59897232055664, "objective/scores": 16.875, "policy/approxkl_avg": 0.0008458065567538142, "policy/clipfrac_avg": 0.005439429543912411, "loss/policy_avg": -0.21535149216651917, "loss/value_avg": 0.42513978481292725, "val/clipfrac_avg": 0.03808614984154701, "policy/entropy_avg": 0.16998863220214844, "val/ratio": 0.9894691109657288, "val/ratio_var": 6.017706618877128e-05, "val/num_eos_tokens": 93, "lr": 2.305996040265198e-06, "episode": 13824, "epoch": 0.6981818181818182, "step": 864}, {"eps": 0, "objective/kl": 39.21794891357422, "objective/entropy": -82.54826354980469, "objective/non_score_reward": -1.9608973264694214, "objective/rlhf_reward": 14.359415054321289, "objective/scores": 16.375, "policy/approxkl_avg": 0.0007373163243755698, "policy/clipfrac_avg": 0.003970882389694452, "loss/policy_avg": -0.08341407775878906, "loss/value_avg": 0.7828294038772583, "val/clipfrac_avg": 0.17675819993019104, "policy/entropy_avg": 0.14882469177246094, "val/ratio": 0.9905297756195068, "val/ratio_var": 5.841355596203357e-05, "val/num_eos_tokens": 134, "lr": 2.29475319215209e-06, "episode": 13840, "epoch": 0.6989898989898989, "step": 865}, {"eps": 0, "objective/kl": 34.1153564453125, "objective/entropy": -121.09165954589844, "objective/non_score_reward": -1.7057678699493408, "objective/rlhf_reward": 13.333294868469238, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0009264532127417624, "policy/clipfrac_avg": 0.0050398847088217735, "loss/policy_avg": 0.029253050684928894, "loss/value_avg": 0.8540998101234436, "val/clipfrac_avg": 0.02329857461154461, "policy/entropy_avg": 0.18099594116210938, "val/ratio": 0.9883856177330017, "val/ratio_var": 8.088548202067614e-05, "val/num_eos_tokens": 1353, "lr": 2.2835296500138664e-06, "episode": 13856, "epoch": 0.6997979797979798, "step": 866}, {"eps": 0, "objective/kl": 36.36531066894531, "objective/entropy": -49.65830993652344, "objective/non_score_reward": -1.8182655572891235, "objective/rlhf_reward": 14.494234085083008, "objective/scores": 16.25, "policy/approxkl_avg": 0.0010429773246869445, "policy/clipfrac_avg": 0.005474097095429897, "loss/policy_avg": -0.1498204916715622, "loss/value_avg": 0.4047302007675171, "val/clipfrac_avg": 0.0032526040449738503, "policy/entropy_avg": 0.18477344512939453, "val/ratio": 0.9877700805664062, "val/ratio_var": 9.155944280792028e-05, "val/num_eos_tokens": 101, "lr": 2.272325493947257e-06, "episode": 13872, "epoch": 0.7006060606060606, "step": 867}, {"eps": 0, "objective/kl": 38.648643493652344, "objective/entropy": -77.34773254394531, "objective/non_score_reward": -1.9324320554733276, "objective/rlhf_reward": 13.637880325317383, "objective/scores": 15.5625, "policy/approxkl_avg": 0.000937658129259944, "policy/clipfrac_avg": 0.006235727574676275, "loss/policy_avg": -0.1165279895067215, "loss/value_avg": 0.7669879198074341, "val/clipfrac_avg": 0.058441728353500366, "policy/entropy_avg": 0.19645977020263672, "val/ratio": 0.9879204034805298, "val/ratio_var": 6.414015660993755e-05, "val/num_eos_tokens": 310, "lr": 2.261140803910644e-06, "episode": 13888, "epoch": 0.7014141414141414, "step": 868}, {"eps": 0, "objective/kl": 48.81157684326172, "objective/entropy": -163.28765869140625, "objective/non_score_reward": -2.4405789375305176, "objective/rlhf_reward": 11.70395278930664, "objective/scores": 14.125, "policy/approxkl_avg": 0.000844982685521245, "policy/clipfrac_avg": 0.0032518147490918636, "loss/policy_avg": -0.016327757388353348, "loss/value_avg": 0.43844515085220337, "val/clipfrac_avg": 0.014801801182329655, "policy/entropy_avg": 0.16161727905273438, "val/ratio": 0.9885438680648804, "val/ratio_var": 5.214377597440034e-05, "val/num_eos_tokens": 1949, "lr": 2.2499756597234872e-06, "episode": 13904, "epoch": 0.7022222222222222, "step": 869}, {"eps": 0, "objective/kl": 40.3735237121582, "objective/entropy": -90.82677459716797, "objective/non_score_reward": -2.0186760425567627, "objective/rlhf_reward": 15.153199195861816, "objective/scores": 17.125, "policy/approxkl_avg": 0.0008435281342826784, "policy/clipfrac_avg": 0.005789198447018862, "loss/policy_avg": -0.18964827060699463, "loss/value_avg": 0.5136280655860901, "val/clipfrac_avg": 0.0648677870631218, "policy/entropy_avg": 0.1680126190185547, "val/ratio": 0.9890276193618774, "val/ratio_var": 7.125848060240969e-05, "val/num_eos_tokens": 555, "lr": 2.238830141065765e-06, "episode": 13920, "epoch": 0.703030303030303, "step": 870}, {"eps": 0, "objective/kl": 39.21965026855469, "objective/entropy": -45.07421112060547, "objective/non_score_reward": -1.9609825611114502, "objective/rlhf_reward": 14.624954223632812, "objective/scores": 16.625, "policy/approxkl_avg": 0.0009133699350059032, "policy/clipfrac_avg": 0.004030972719192505, "loss/policy_avg": 0.02698252536356449, "loss/value_avg": 0.2791609764099121, "val/clipfrac_avg": 0.03648557886481285, "policy/entropy_avg": 0.18420028686523438, "val/ratio": 0.9881106615066528, "val/ratio_var": 5.0894170271931216e-05, "val/num_eos_tokens": 232, "lr": 2.2277043274773856e-06, "episode": 13936, "epoch": 0.7038383838383838, "step": 871}, {"eps": 0, "objective/kl": 37.084686279296875, "objective/entropy": -52.43424987792969, "objective/non_score_reward": -1.8542344570159912, "objective/rlhf_reward": 14.18482780456543, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008421384263783693, "policy/clipfrac_avg": 0.002664695493876934, "loss/policy_avg": -0.12599462270736694, "loss/value_avg": 0.5007761716842651, "val/clipfrac_avg": 0.0010017228778451681, "policy/entropy_avg": 0.1716923713684082, "val/ratio": 0.9881088137626648, "val/ratio_var": 6.610793934669346e-05, "val/num_eos_tokens": 274, "lr": 2.216598298357646e-06, "episode": 13952, "epoch": 0.7046464646464646, "step": 872}, {"eps": 0, "objective/kl": 41.75783157348633, "objective/entropy": -138.92434692382812, "objective/non_score_reward": -2.0878915786743164, "objective/rlhf_reward": 13.091796875, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0008433774346485734, "policy/clipfrac_avg": 0.006052501033991575, "loss/policy_avg": 0.35178622603416443, "loss/value_avg": 0.5846590995788574, "val/clipfrac_avg": 0.0322103314101696, "policy/entropy_avg": 0.16450119018554688, "val/ratio": 0.9888837337493896, "val/ratio_var": 5.522933497559279e-05, "val/num_eos_tokens": 1144, "lr": 2.2055121329646416e-06, "episode": 13968, "epoch": 0.7054545454545454, "step": 873}, {"eps": 0, "objective/kl": 37.81830978393555, "objective/entropy": -122.14696502685547, "objective/non_score_reward": -1.8909155130386353, "objective/rlhf_reward": 13.296585083007812, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0006640740903094411, "policy/clipfrac_avg": 0.004708480089902878, "loss/policy_avg": -0.004525661468505859, "loss/value_avg": 0.5259770750999451, "val/clipfrac_avg": 0.10728074610233307, "policy/entropy_avg": 0.14041614532470703, "val/ratio": 0.990477979183197, "val/ratio_var": 6.866270996397361e-05, "val/num_eos_tokens": 35, "lr": 2.1944459104147188e-06, "episode": 13984, "epoch": 0.7062626262626263, "step": 874}, {"eps": 0, "objective/kl": 37.62938690185547, "objective/entropy": -54.207332611083984, "objective/non_score_reward": -1.8814692497253418, "objective/rlhf_reward": 14.2200927734375, "objective/scores": 16.125, "policy/approxkl_avg": 0.000859625346492976, "policy/clipfrac_avg": 0.005390159785747528, "loss/policy_avg": 0.11862924695014954, "loss/value_avg": 1.0722770690917969, "val/clipfrac_avg": 0.0593750923871994, "policy/entropy_avg": 0.1755199432373047, "val/ratio": 0.9878208637237549, "val/ratio_var": 6.484645564341918e-05, "val/num_eos_tokens": 201, "lr": 2.1833997096818897e-06, "episode": 14000, "epoch": 0.7070707070707071, "step": 875}, {"eps": 0, "objective/kl": 44.697776794433594, "objective/entropy": -16.795242309570312, "objective/non_score_reward": -2.234889030456543, "objective/rlhf_reward": 13.140110969543457, "objective/scores": 15.375, "policy/approxkl_avg": 0.0011914006900042295, "policy/clipfrac_avg": 0.005575358867645264, "loss/policy_avg": -0.2030085027217865, "loss/value_avg": 0.5594953298568726, "val/clipfrac_avg": 0.02393021248281002, "policy/entropy_avg": 0.2543792724609375, "val/ratio": 0.9835648536682129, "val/ratio_var": 0.00010361165914218873, "val/num_eos_tokens": 24, "lr": 2.1723736095972946e-06, "episode": 14016, "epoch": 0.7078787878787879, "step": 876}, {"eps": 0, "objective/kl": 50.78942108154297, "objective/entropy": -105.85357666015625, "objective/non_score_reward": -2.53947114944458, "objective/rlhf_reward": 13.077716827392578, "objective/scores": 15.625, "policy/approxkl_avg": 0.0008386933477595448, "policy/clipfrac_avg": 0.003330180887132883, "loss/policy_avg": 0.05069049820303917, "loss/value_avg": 0.6454532146453857, "val/clipfrac_avg": 0.010819870047271252, "policy/entropy_avg": 0.1841726303100586, "val/ratio": 0.9886084198951721, "val/ratio_var": 5.4145024478202686e-05, "val/num_eos_tokens": 513, "lr": 2.161367688848618e-06, "episode": 14032, "epoch": 0.7086868686868687, "step": 877}, {"eps": 0, "objective/kl": 41.63684844970703, "objective/entropy": -193.71432495117188, "objective/non_score_reward": -2.0818424224853516, "objective/rlhf_reward": 14.261907577514648, "objective/scores": 16.375, "policy/approxkl_avg": 0.0007008577231317759, "policy/clipfrac_avg": 0.004103646147996187, "loss/policy_avg": -0.32361558079719543, "loss/value_avg": 0.4275820851325989, "val/clipfrac_avg": 7.62939453125e-06, "policy/entropy_avg": 0.1445927619934082, "val/ratio": 0.9903497695922852, "val/ratio_var": 3.22636442433577e-05, "val/num_eos_tokens": 323, "lr": 2.1503820259795357e-06, "episode": 14048, "epoch": 0.7094949494949495, "step": 878}, {"eps": 0, "objective/kl": 34.282958984375, "objective/entropy": -45.89549255371094, "objective/non_score_reward": -1.7141478061676025, "objective/rlhf_reward": 14.692102432250977, "objective/scores": 16.375, "policy/approxkl_avg": 0.0007650890620425344, "policy/clipfrac_avg": 0.00561137031763792, "loss/policy_avg": -0.1645442247390747, "loss/value_avg": 0.7777354717254639, "val/clipfrac_avg": 0.10208053886890411, "policy/entropy_avg": 0.15566253662109375, "val/ratio": 0.9895084500312805, "val/ratio_var": 3.380152702447958e-05, "val/num_eos_tokens": 49, "lr": 2.139416699389153e-06, "episode": 14064, "epoch": 0.7103030303030303, "step": 879}, {"eps": 0, "objective/kl": 41.30862808227539, "objective/entropy": -50.96327209472656, "objective/non_score_reward": -2.0654313564300537, "objective/rlhf_reward": 14.450193405151367, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009377336828038096, "policy/clipfrac_avg": 0.00421791011467576, "loss/policy_avg": -0.023053357377648354, "loss/value_avg": 0.521371603012085, "val/clipfrac_avg": 0.04448426887392998, "policy/entropy_avg": 0.17507076263427734, "val/ratio": 0.9889349937438965, "val/ratio_var": 3.734078927664086e-05, "val/num_eos_tokens": 368, "lr": 2.128471787331446e-06, "episode": 14080, "epoch": 0.7111111111111111, "step": 880}, {"eps": 0, "objective/kl": 41.05116653442383, "objective/entropy": -69.7264175415039, "objective/non_score_reward": -2.052558422088623, "objective/rlhf_reward": 14.830253601074219, "objective/scores": 16.875, "policy/approxkl_avg": 0.001040782080963254, "policy/clipfrac_avg": 0.0033751465380191803, "loss/policy_avg": -0.07007278501987457, "loss/value_avg": 0.2162141352891922, "val/clipfrac_avg": 0.03193364292383194, "policy/entropy_avg": 0.20317459106445312, "val/ratio": 0.9861003160476685, "val/ratio_var": 8.361194340977818e-05, "val/num_eos_tokens": 551, "lr": 2.117547367914705e-06, "episode": 14096, "epoch": 0.711919191919192, "step": 881}, {"eps": 0, "objective/kl": 45.00962829589844, "objective/entropy": -86.86426544189453, "objective/non_score_reward": -2.250481367111206, "objective/rlhf_reward": 13.913580894470215, "objective/scores": 16.125, "policy/approxkl_avg": 0.0010606081923469901, "policy/clipfrac_avg": 0.0069802612997591496, "loss/policy_avg": -0.07847104221582413, "loss/value_avg": 0.2657424807548523, "val/clipfrac_avg": 0.038475796580314636, "policy/entropy_avg": 0.20689773559570312, "val/ratio": 0.985522449016571, "val/ratio_var": 8.580441499361768e-05, "val/num_eos_tokens": 546, "lr": 2.1066435191009717e-06, "episode": 14112, "epoch": 0.7127272727272728, "step": 882}, {"eps": 0, "objective/kl": 42.61073303222656, "objective/entropy": -109.19290161132812, "objective/non_score_reward": -2.1305365562438965, "objective/rlhf_reward": 14.447587966918945, "objective/scores": 16.625, "policy/approxkl_avg": 0.0010359378065913916, "policy/clipfrac_avg": 0.006892787292599678, "loss/policy_avg": -0.1323227733373642, "loss/value_avg": 0.21754738688468933, "val/clipfrac_avg": 0.010656789876520634, "policy/entropy_avg": 0.20017433166503906, "val/ratio": 0.9864689111709595, "val/ratio_var": 4.844724026042968e-05, "val/num_eos_tokens": 36, "lr": 2.095760318705487e-06, "episode": 14128, "epoch": 0.7135353535353536, "step": 883}, {"eps": 0, "objective/kl": 43.517696380615234, "objective/entropy": -119.52471923828125, "objective/non_score_reward": -2.17588472366333, "objective/rlhf_reward": 14.300678253173828, "objective/scores": 16.5, "policy/approxkl_avg": 0.000938809709623456, "policy/clipfrac_avg": 0.012131594121456146, "loss/policy_avg": -0.1507091522216797, "loss/value_avg": 0.2665415108203888, "val/clipfrac_avg": 0.006075765937566757, "policy/entropy_avg": 0.18040943145751953, "val/ratio": 0.9873965978622437, "val/ratio_var": 6.621635111514479e-05, "val/num_eos_tokens": 620, "lr": 2.0848978443961314e-06, "episode": 14144, "epoch": 0.7143434343434344, "step": 884}, {"eps": 0, "objective/kl": 45.77116394042969, "objective/entropy": -62.273380279541016, "objective/non_score_reward": -2.2885584831237793, "objective/rlhf_reward": 14.617691040039062, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009718666551634669, "policy/clipfrac_avg": 0.004924894310534, "loss/policy_avg": -0.028530046343803406, "loss/value_avg": 0.434810072183609, "val/clipfrac_avg": 0.036309417337179184, "policy/entropy_avg": 0.20779800415039062, "val/ratio": 0.9870821237564087, "val/ratio_var": 8.347785478690639e-05, "val/num_eos_tokens": 179, "lr": 2.074056173692881e-06, "episode": 14160, "epoch": 0.7151515151515152, "step": 885}, {"eps": 0, "objective/kl": 32.41828918457031, "objective/entropy": -84.31056213378906, "objective/non_score_reward": -1.6209145784378052, "objective/rlhf_reward": 15.332210540771484, "objective/scores": 17.0, "policy/approxkl_avg": 0.0008589382632635534, "policy/clipfrac_avg": 0.002752305706962943, "loss/policy_avg": -0.15033547580242157, "loss/value_avg": 0.6043248772621155, "val/clipfrac_avg": 0.03947104513645172, "policy/entropy_avg": 0.18497180938720703, "val/ratio": 0.9879313707351685, "val/ratio_var": 5.601351585937664e-05, "val/num_eos_tokens": 180, "lr": 2.0632353839672415e-06, "episode": 14176, "epoch": 0.7159595959595959, "step": 886}, {"eps": 0, "objective/kl": 36.84368133544922, "objective/entropy": -51.082984924316406, "objective/non_score_reward": -1.842184066772461, "objective/rlhf_reward": 14.087503433227539, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0010461971396580338, "policy/clipfrac_avg": 0.004587469156831503, "loss/policy_avg": 0.2279771864414215, "loss/value_avg": 1.2074341773986816, "val/clipfrac_avg": 0.015519710257649422, "policy/entropy_avg": 0.22597980499267578, "val/ratio": 0.9849185943603516, "val/ratio_var": 6.34926327620633e-05, "val/num_eos_tokens": 448, "lr": 2.0524355524417017e-06, "episode": 14192, "epoch": 0.7167676767676767, "step": 887}, {"eps": 0, "objective/kl": 42.720699310302734, "objective/entropy": -154.47012329101562, "objective/non_score_reward": -2.1360349655151367, "objective/rlhf_reward": 14.957715034484863, "objective/scores": 17.125, "policy/approxkl_avg": 0.0009299394441768527, "policy/clipfrac_avg": 0.008321288973093033, "loss/policy_avg": -0.026267291978001595, "loss/value_avg": 0.5965927839279175, "val/clipfrac_avg": 0.01875000074505806, "policy/entropy_avg": 0.1751893162727356, "val/ratio": 0.9883604049682617, "val/ratio_var": 7.325677870539948e-05, "val/num_eos_tokens": 428, "lr": 2.041656756189184e-06, "episode": 14208, "epoch": 0.7175757575757575, "step": 888}, {"eps": 0, "objective/kl": 33.898155212402344, "objective/entropy": -128.48831176757812, "objective/non_score_reward": -1.6949079036712646, "objective/rlhf_reward": 12.598060607910156, "objective/scores": 14.3125, "policy/approxkl_avg": 0.0006705562118440866, "policy/clipfrac_avg": 0.005831941030919552, "loss/policy_avg": 0.40078702569007874, "loss/value_avg": 1.1534854173660278, "val/clipfrac_avg": 0.02483724057674408, "policy/entropy_avg": 0.21662330627441406, "val/ratio": 0.9912885427474976, "val/ratio_var": 7.058407936710864e-05, "val/num_eos_tokens": 47, "lr": 2.030899072132493e-06, "episode": 14224, "epoch": 0.7183838383838383, "step": 889}, {"eps": 0, "objective/kl": 40.06652069091797, "objective/entropy": -86.9806137084961, "objective/non_score_reward": -2.003325939178467, "objective/rlhf_reward": 14.035737037658691, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008815613109618425, "policy/clipfrac_avg": 0.006017718929797411, "loss/policy_avg": -0.0855930745601654, "loss/value_avg": 0.32921552658081055, "val/clipfrac_avg": 0.06256753951311111, "policy/entropy_avg": 0.1755352020263672, "val/ratio": 0.9872984290122986, "val/ratio_var": 8.527067984687164e-05, "val/num_eos_tokens": 20, "lr": 2.0201625770437647e-06, "episode": 14240, "epoch": 0.7191919191919192, "step": 890}, {"eps": 0, "objective/kl": 41.95650863647461, "objective/entropy": -99.16207122802734, "objective/non_score_reward": -2.097825288772583, "objective/rlhf_reward": 13.74592399597168, "objective/scores": 15.875, "policy/approxkl_avg": 0.0007897773757576942, "policy/clipfrac_avg": 0.0043031517416238785, "loss/policy_avg": -0.17781905829906464, "loss/value_avg": 0.2275725156068802, "val/clipfrac_avg": 0.012334126979112625, "policy/entropy_avg": 0.1585559844970703, "val/ratio": 0.9894897937774658, "val/ratio_var": 5.430886085378006e-05, "val/num_eos_tokens": 424, "lr": 2.00944734754392e-06, "episode": 14256, "epoch": 0.72, "step": 891}, {"eps": 0, "objective/kl": 37.07469177246094, "objective/entropy": -61.42802047729492, "objective/non_score_reward": -1.8537347316741943, "objective/rlhf_reward": 13.044702529907227, "objective/scores": 14.875, "policy/approxkl_avg": 0.0008486385922878981, "policy/clipfrac_avg": 0.006757801398634911, "loss/policy_avg": -0.36396488547325134, "loss/value_avg": 0.8662234544754028, "val/clipfrac_avg": 0.054801926016807556, "policy/entropy_avg": 0.1866321563720703, "val/ratio": 0.9883260726928711, "val/ratio_var": 6.7303488322068e-05, "val/num_eos_tokens": 584, "lr": 1.9987534601021225e-06, "episode": 14272, "epoch": 0.7208080808080808, "step": 892}, {"eps": 0, "objective/kl": 42.03372573852539, "objective/entropy": -58.38874053955078, "objective/non_score_reward": -2.1016862392425537, "objective/rlhf_reward": 12.828001022338867, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0008078413084149361, "policy/clipfrac_avg": 0.004710954613983631, "loss/policy_avg": -0.12171509861946106, "loss/value_avg": 0.3499939441680908, "val/clipfrac_avg": 0.03385913744568825, "policy/entropy_avg": 0.16219139099121094, "val/ratio": 0.9909918904304504, "val/ratio_var": 4.974031253368594e-05, "val/num_eos_tokens": 211, "lr": 1.988080991035223e-06, "episode": 14288, "epoch": 0.7216161616161616, "step": 893}, {"eps": 0, "objective/kl": 43.86260986328125, "objective/entropy": -72.03929138183594, "objective/non_score_reward": -2.1931304931640625, "objective/rlhf_reward": 13.322494506835938, "objective/scores": 15.5, "policy/approxkl_avg": 0.0009859772399067879, "policy/clipfrac_avg": 0.004861406981945038, "loss/policy_avg": 0.15615974366664886, "loss/value_avg": 0.4985978901386261, "val/clipfrac_avg": 0.008869262412190437, "policy/entropy_avg": 0.19806480407714844, "val/ratio": 0.9869673252105713, "val/ratio_var": 5.9525649703573436e-05, "val/num_eos_tokens": 389, "lr": 1.977430016507222e-06, "episode": 14304, "epoch": 0.7224242424242424, "step": 894}, {"eps": 0, "objective/kl": 38.795677185058594, "objective/entropy": -30.8804874420166, "objective/non_score_reward": -1.9397838115692139, "objective/rlhf_reward": 12.958653450012207, "objective/scores": 14.875, "policy/approxkl_avg": 0.0011067797895520926, "policy/clipfrac_avg": 0.004753993824124336, "loss/policy_avg": -0.018653089180588722, "loss/value_avg": 0.3236011266708374, "val/clipfrac_avg": 0.013285137712955475, "policy/entropy_avg": 0.21401214599609375, "val/ratio": 0.9844212532043457, "val/ratio_var": 5.578770287684165e-05, "val/num_eos_tokens": 190, "lr": 1.966800612528723e-06, "episode": 14320, "epoch": 0.7232323232323232, "step": 895}, {"eps": 0, "objective/kl": 40.9593505859375, "objective/entropy": -88.01570129394531, "objective/non_score_reward": -2.0479674339294434, "objective/rlhf_reward": 13.350469589233398, "objective/scores": 15.375, "policy/approxkl_avg": 0.0010309319477528334, "policy/clipfrac_avg": 0.005483411252498627, "loss/policy_avg": -0.04005177319049835, "loss/value_avg": 0.564488410949707, "val/clipfrac_avg": 0.01046415138989687, "policy/entropy_avg": 0.21037864685058594, "val/ratio": 0.9865807890892029, "val/ratio_var": 9.776368096936494e-05, "val/num_eos_tokens": 539, "lr": 1.956192854956397e-06, "episode": 14336, "epoch": 0.724040404040404, "step": 896}, {"eps": 0, "objective/kl": 37.47423553466797, "objective/entropy": -98.2094955444336, "objective/non_score_reward": -1.8737118244171143, "objective/rlhf_reward": 13.634100914001465, "objective/scores": 15.5, "policy/approxkl_avg": 0.0007985340780578554, "policy/clipfrac_avg": 0.005063176155090332, "loss/policy_avg": -0.1358407437801361, "loss/value_avg": 0.536889910697937, "val/clipfrac_avg": 0.06062069162726402, "policy/entropy_avg": 0.14582359790802002, "val/ratio": 0.9903557300567627, "val/ratio_var": 5.06751166540198e-05, "val/num_eos_tokens": 1182, "lr": 1.945606819492429e-06, "episode": 14352, "epoch": 0.7248484848484849, "step": 897}, {"eps": 0, "objective/kl": 41.13695526123047, "objective/entropy": -110.30941772460938, "objective/non_score_reward": -2.0568478107452393, "objective/rlhf_reward": 13.72440242767334, "objective/scores": 15.75, "policy/approxkl_avg": 0.0011626393534243107, "policy/clipfrac_avg": 0.006754214409738779, "loss/policy_avg": 0.011427253484725952, "loss/value_avg": 0.263162761926651, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2226581573486328, "val/ratio": 0.9850178956985474, "val/ratio_var": 8.876781794242561e-05, "val/num_eos_tokens": 10, "lr": 1.9350425816839875e-06, "episode": 14368, "epoch": 0.7256565656565657, "step": 898}, {"eps": 0, "objective/kl": 48.557960510253906, "objective/entropy": -101.70489501953125, "objective/non_score_reward": -2.4278979301452637, "objective/rlhf_reward": 13.314289093017578, "objective/scores": 15.75, "policy/approxkl_avg": 0.0009826980531215668, "policy/clipfrac_avg": 0.004610124975442886, "loss/policy_avg": 0.11541193723678589, "loss/value_avg": 0.4631091356277466, "val/clipfrac_avg": 0.013620924204587936, "policy/entropy_avg": 0.1724681854248047, "val/ratio": 0.9876973628997803, "val/ratio_var": 5.4934793297434226e-05, "val/num_eos_tokens": 516, "lr": 1.9245002169226814e-06, "episode": 14384, "epoch": 0.7264646464646465, "step": 899}, {"eps": 0, "objective/kl": 41.26939010620117, "objective/entropy": -91.96019744873047, "objective/non_score_reward": -2.063469409942627, "objective/rlhf_reward": 12.522467613220215, "objective/scores": 14.5625, "policy/approxkl_avg": 0.0009332017507404089, "policy/clipfrac_avg": 0.0043076397851109505, "loss/policy_avg": 0.07936800271272659, "loss/value_avg": 0.6616516709327698, "val/clipfrac_avg": 0.01476163137704134, "policy/entropy_avg": 0.18123245239257812, "val/ratio": 0.9875973463058472, "val/ratio_var": 7.475823076674715e-05, "val/num_eos_tokens": 595, "lr": 1.913979800444021e-06, "episode": 14400, "epoch": 0.7272727272727273, "step": 900}, {"eps": 0, "objective/kl": 39.73054122924805, "objective/entropy": -59.236602783203125, "objective/non_score_reward": -1.9865270853042603, "objective/rlhf_reward": 12.435348510742188, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0011704354546964169, "policy/clipfrac_avg": 0.004937891382724047, "loss/policy_avg": -0.02671997621655464, "loss/value_avg": 0.7174835205078125, "val/clipfrac_avg": 0.13261640071868896, "policy/entropy_avg": 0.22887516021728516, "val/ratio": 0.9848372936248779, "val/ratio_var": 8.883672853698954e-05, "val/num_eos_tokens": 121, "lr": 1.9034814073268898e-06, "episode": 14416, "epoch": 0.7280808080808081, "step": 901}, {"eps": 0, "objective/kl": 46.268035888671875, "objective/entropy": -53.46650695800781, "objective/non_score_reward": -2.313401937484741, "objective/rlhf_reward": 14.38972282409668, "objective/scores": 16.75, "policy/approxkl_avg": 0.0010022292844951153, "policy/clipfrac_avg": 0.004066841676831245, "loss/policy_avg": 0.05788304656744003, "loss/value_avg": 0.2591496407985687, "val/clipfrac_avg": 0.013965151272714138, "policy/entropy_avg": 0.20316123962402344, "val/ratio": 0.9866280555725098, "val/ratio_var": 5.229823727859184e-05, "val/num_eos_tokens": 455, "lr": 1.8930051124929904e-06, "episode": 14432, "epoch": 0.7288888888888889, "step": 902}, {"eps": 0, "objective/kl": 46.209991455078125, "objective/entropy": -65.6883773803711, "objective/non_score_reward": -2.310499668121338, "objective/rlhf_reward": 16.78325080871582, "objective/scores": 19.125, "policy/approxkl_avg": 0.0010066109243780375, "policy/clipfrac_avg": 0.005406823009252548, "loss/policy_avg": -0.11720985174179077, "loss/value_avg": 0.23323997855186462, "val/clipfrac_avg": 0.015051587484776974, "policy/entropy_avg": 0.20101547241210938, "val/ratio": 0.9865299463272095, "val/ratio_var": 5.01565846207086e-05, "val/num_eos_tokens": 205, "lr": 1.8825509907063328e-06, "episode": 14448, "epoch": 0.7296969696969697, "step": 903}, {"eps": 0, "objective/kl": 37.29925537109375, "objective/entropy": -118.0733642578125, "objective/non_score_reward": -1.8649628162384033, "objective/rlhf_reward": 15.885037422180176, "objective/scores": 17.75, "policy/approxkl_avg": 0.0007588897133246064, "policy/clipfrac_avg": 0.0023906652349978685, "loss/policy_avg": -0.4681462049484253, "loss/value_avg": 0.8052223920822144, "val/clipfrac_avg": 0.13810205459594727, "policy/entropy_avg": 0.15973949432373047, "val/ratio": 0.9902135729789734, "val/ratio_var": 7.25651771062985e-05, "val/num_eos_tokens": 155, "lr": 1.87211911657268e-06, "episode": 14464, "epoch": 0.7305050505050505, "step": 904}, {"eps": 0, "objective/kl": 39.160484313964844, "objective/entropy": -22.496440887451172, "objective/non_score_reward": -1.958024263381958, "objective/rlhf_reward": 12.323225975036621, "objective/scores": 14.25, "policy/approxkl_avg": 0.0010718535631895065, "policy/clipfrac_avg": 0.004276500083506107, "loss/policy_avg": 0.19160020351409912, "loss/value_avg": 0.9071890115737915, "val/clipfrac_avg": 0.03053325042128563, "policy/entropy_avg": 0.19223785400390625, "val/ratio": 0.9871090650558472, "val/ratio_var": 4.5721961214439943e-05, "val/num_eos_tokens": 240, "lr": 1.8617095645390332e-06, "episode": 14480, "epoch": 0.7313131313131314, "step": 905}, {"eps": 0, "objective/kl": 30.631162643432617, "objective/entropy": -52.41043472290039, "objective/non_score_reward": -1.5315582752227783, "objective/rlhf_reward": 16.296566009521484, "objective/scores": 17.875, "policy/approxkl_avg": 0.001107763615436852, "policy/clipfrac_avg": 0.006524171680212021, "loss/policy_avg": -0.11877916008234024, "loss/value_avg": 0.6873422265052795, "val/clipfrac_avg": 0.000294811325147748, "policy/entropy_avg": 0.2064342498779297, "val/ratio": 0.9861389398574829, "val/ratio_var": 6.195842433953658e-05, "val/num_eos_tokens": 261, "lr": 1.8513224088930814e-06, "episode": 14496, "epoch": 0.7321212121212122, "step": 906}, {"eps": 0, "objective/kl": 43.33234405517578, "objective/entropy": -61.856544494628906, "objective/non_score_reward": -2.1666173934936523, "objective/rlhf_reward": 13.145882606506348, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0010527942795306444, "policy/clipfrac_avg": 0.004575061611831188, "loss/policy_avg": 0.10565789043903351, "loss/value_avg": 0.5067005157470703, "val/clipfrac_avg": 0.005580357275903225, "policy/entropy_avg": 0.18899154663085938, "val/ratio": 0.9872788190841675, "val/ratio_var": 7.066375110298395e-05, "val/num_eos_tokens": 496, "lr": 1.8409577237626935e-06, "episode": 14512, "epoch": 0.7329292929292929, "step": 907}, {"eps": 0, "objective/kl": 37.619895935058594, "objective/entropy": -91.67915344238281, "objective/non_score_reward": -1.8809947967529297, "objective/rlhf_reward": 14.22056770324707, "objective/scores": 16.125, "policy/approxkl_avg": 0.0009494669502601027, "policy/clipfrac_avg": 0.007099117152392864, "loss/policy_avg": 0.04191252216696739, "loss/value_avg": 0.30707108974456787, "val/clipfrac_avg": 0.07242873311042786, "policy/entropy_avg": 0.17319107055664062, "val/ratio": 0.9883796572685242, "val/ratio_var": 7.36714864615351e-05, "val/num_eos_tokens": 437, "lr": 1.8306155831153699e-06, "episode": 14528, "epoch": 0.7337373737373737, "step": 908}, {"eps": 0, "objective/kl": 36.91444396972656, "objective/entropy": -92.03844451904297, "objective/non_score_reward": -1.8457221984863281, "objective/rlhf_reward": 13.802715301513672, "objective/scores": 15.625, "policy/approxkl_avg": 0.0009770962642505765, "policy/clipfrac_avg": 0.005569060333073139, "loss/policy_avg": 0.15317559242248535, "loss/value_avg": 0.7379442453384399, "val/clipfrac_avg": 0.04051949828863144, "policy/entropy_avg": 0.18958091735839844, "val/ratio": 0.988923192024231, "val/ratio_var": 6.142368511063978e-05, "val/num_eos_tokens": 56, "lr": 1.8202960607577246e-06, "episode": 14544, "epoch": 0.7345454545454545, "step": 909}, {"eps": 0, "objective/kl": 38.83802032470703, "objective/entropy": -111.80148315429688, "objective/non_score_reward": -1.9419012069702148, "objective/rlhf_reward": 14.401848793029785, "objective/scores": 16.375, "policy/approxkl_avg": 0.000904000480659306, "policy/clipfrac_avg": 0.005316127557307482, "loss/policy_avg": 0.1602431833744049, "loss/value_avg": 0.3025469183921814, "val/clipfrac_avg": 0.00018382353300694376, "policy/entropy_avg": 0.17273902893066406, "val/ratio": 0.9891130924224854, "val/ratio_var": 7.234779332065955e-05, "val/num_eos_tokens": 342, "lr": 1.809999230334958e-06, "episode": 14560, "epoch": 0.7353535353535353, "step": 910}, {"eps": 0, "objective/kl": 44.55670166015625, "objective/entropy": -98.02986145019531, "objective/non_score_reward": -2.227834939956665, "objective/rlhf_reward": 13.092477798461914, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0009336632210761309, "policy/clipfrac_avg": 0.004822542425245047, "loss/policy_avg": 0.09882547706365585, "loss/value_avg": 0.6121327877044678, "val/clipfrac_avg": 0.0050772735849022865, "policy/entropy_avg": 0.17920780181884766, "val/ratio": 0.9875223636627197, "val/ratio_var": 6.380004924722016e-05, "val/num_eos_tokens": 426, "lr": 1.7997251653303249e-06, "episode": 14576, "epoch": 0.7361616161616161, "step": 911}, {"eps": 0, "objective/kl": 36.05466079711914, "objective/entropy": -57.16059875488281, "objective/non_score_reward": -1.802733063697815, "objective/rlhf_reward": 13.119142532348633, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0009670511353760958, "policy/clipfrac_avg": 0.00586615689098835, "loss/policy_avg": -0.15951846539974213, "loss/value_avg": 0.5648845434188843, "val/clipfrac_avg": 0.0008223684271797538, "policy/entropy_avg": 0.1721019744873047, "val/ratio": 0.9892112016677856, "val/ratio_var": 4.9591733841225505e-05, "val/num_eos_tokens": 45, "lr": 1.7894739390646227e-06, "episode": 14592, "epoch": 0.7369696969696969, "step": 912}, {"eps": 0, "objective/kl": 46.18011474609375, "objective/entropy": -146.6658935546875, "objective/non_score_reward": -2.3090057373046875, "objective/rlhf_reward": 14.511306762695312, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009307245491072536, "policy/clipfrac_avg": 0.006143895909190178, "loss/policy_avg": 0.1940194070339203, "loss/value_avg": 0.3815060555934906, "val/clipfrac_avg": 0.0180903859436512, "policy/entropy_avg": 0.18556424975395203, "val/ratio": 0.9874189496040344, "val/ratio_var": 0.00011522904242156073, "val/num_eos_tokens": 698, "lr": 1.7792456246956535e-06, "episode": 14608, "epoch": 0.7377777777777778, "step": 913}, {"eps": 0, "objective/kl": 35.78252410888672, "objective/entropy": -84.84890747070312, "objective/non_score_reward": -1.7891263961791992, "objective/rlhf_reward": 13.2655611038208, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0009321504039689898, "policy/clipfrac_avg": 0.005007730796933174, "loss/policy_avg": -0.02095976285636425, "loss/value_avg": 0.7224372625350952, "val/clipfrac_avg": 0.023768942803144455, "policy/entropy_avg": 0.18702125549316406, "val/ratio": 0.9874284267425537, "val/ratio_var": 8.100390550680459e-05, "val/num_eos_tokens": 403, "lr": 1.7690402952177117e-06, "episode": 14624, "epoch": 0.7385858585858586, "step": 914}, {"eps": 0, "objective/kl": 38.5015869140625, "objective/entropy": -63.22113037109375, "objective/non_score_reward": -1.925079345703125, "objective/rlhf_reward": 13.918670654296875, "objective/scores": 15.875, "policy/approxkl_avg": 0.0008501409320160747, "policy/clipfrac_avg": 0.004706401377916336, "loss/policy_avg": -0.3134731650352478, "loss/value_avg": 0.5765621066093445, "val/clipfrac_avg": 0.06931675970554352, "policy/entropy_avg": 0.1600513458251953, "val/ratio": 0.9882992506027222, "val/ratio_var": 6.202542135724798e-05, "val/num_eos_tokens": 171, "lr": 1.7588580234610592e-06, "episode": 14640, "epoch": 0.7393939393939394, "step": 915}, {"eps": 0, "objective/kl": 42.955291748046875, "objective/entropy": -43.271400451660156, "objective/non_score_reward": -2.1477646827697754, "objective/rlhf_reward": 13.875673294067383, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009495788253843784, "policy/clipfrac_avg": 0.005583056248724461, "loss/policy_avg": -0.2850199043750763, "loss/value_avg": 0.8679437637329102, "val/clipfrac_avg": 0.06379853188991547, "policy/entropy_avg": 0.17831039428710938, "val/ratio": 0.9876798391342163, "val/ratio_var": 4.0028760849963874e-05, "val/num_eos_tokens": 103, "lr": 1.7486988820914063e-06, "episode": 14656, "epoch": 0.7402020202020202, "step": 916}, {"eps": 0, "objective/kl": 44.639923095703125, "objective/entropy": -79.67193603515625, "objective/non_score_reward": -2.2319962978363037, "objective/rlhf_reward": 13.463315963745117, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008941494161263108, "policy/clipfrac_avg": 0.006674608215689659, "loss/policy_avg": 0.21622401475906372, "loss/value_avg": 0.22584834694862366, "val/clipfrac_avg": 0.029584797099232674, "policy/entropy_avg": 0.1719951629638672, "val/ratio": 0.9886019825935364, "val/ratio_var": 4.6610963181592524e-05, "val/num_eos_tokens": 50, "lr": 1.7385629436093958e-06, "episode": 14672, "epoch": 0.741010101010101, "step": 917}, {"eps": 0, "objective/kl": 38.6593132019043, "objective/entropy": -68.00552368164062, "objective/non_score_reward": -1.9329655170440674, "objective/rlhf_reward": 13.887346267700195, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0008251155959442258, "policy/clipfrac_avg": 0.005509347654879093, "loss/policy_avg": -0.0006486065685749054, "loss/value_avg": 0.2962718605995178, "val/clipfrac_avg": 0.003826581174507737, "policy/entropy_avg": 0.1643085479736328, "val/ratio": 0.9888970851898193, "val/ratio_var": 3.0508910640492104e-05, "val/num_eos_tokens": 44, "lr": 1.728450280350079e-06, "episode": 14688, "epoch": 0.7418181818181818, "step": 918}, {"eps": 0, "objective/kl": 40.42371368408203, "objective/entropy": -93.26936340332031, "objective/non_score_reward": -2.021185874938965, "objective/rlhf_reward": 15.166314125061035, "objective/scores": 17.25, "policy/approxkl_avg": 0.0009743100963532925, "policy/clipfrac_avg": 0.0028937258757650852, "loss/policy_avg": -0.3462671935558319, "loss/value_avg": 0.7152901887893677, "val/clipfrac_avg": 0.049027521163225174, "policy/entropy_avg": 0.18361568450927734, "val/ratio": 0.9877507090568542, "val/ratio_var": 5.374600732466206e-05, "val/num_eos_tokens": 136, "lr": 1.7183609644824096e-06, "episode": 14704, "epoch": 0.7426262626262626, "step": 919}, {"eps": 0, "objective/kl": 36.17670440673828, "objective/entropy": -81.48149108886719, "objective/non_score_reward": -1.8088352680206299, "objective/rlhf_reward": 12.589601516723633, "objective/scores": 14.375, "policy/approxkl_avg": 0.0008495738729834557, "policy/clipfrac_avg": 0.0038070818409323692, "loss/policy_avg": 0.1274300217628479, "loss/value_avg": 1.230905294418335, "val/clipfrac_avg": 0.007427365053445101, "policy/entropy_avg": 0.16942787170410156, "val/ratio": 0.9902092218399048, "val/ratio_var": 5.2669711294583976e-05, "val/num_eos_tokens": 270, "lr": 1.708295068008718e-06, "episode": 14720, "epoch": 0.7434343434343434, "step": 920}, {"eps": 0, "objective/kl": 42.419307708740234, "objective/entropy": -74.04690551757812, "objective/non_score_reward": -2.1209654808044434, "objective/rlhf_reward": 13.511846542358398, "objective/scores": 15.625, "policy/approxkl_avg": 0.000907333567738533, "policy/clipfrac_avg": 0.004857117310166359, "loss/policy_avg": -0.257866770029068, "loss/value_avg": 0.5942734479904175, "val/clipfrac_avg": 0.043217435479164124, "policy/entropy_avg": 0.18361282348632812, "val/ratio": 0.9877738356590271, "val/ratio_var": 6.433074304368347e-05, "val/num_eos_tokens": 56, "lr": 1.6982526627642043e-06, "episode": 14736, "epoch": 0.7442424242424243, "step": 921}, {"eps": 0, "objective/kl": 35.72700881958008, "objective/entropy": -54.975791931152344, "objective/non_score_reward": -1.7863504886627197, "objective/rlhf_reward": 13.39333724975586, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0010183433769270778, "policy/clipfrac_avg": 0.003768398892134428, "loss/policy_avg": 0.12189102917909622, "loss/value_avg": 0.41360408067703247, "val/clipfrac_avg": 0.0008610248914919794, "policy/entropy_avg": 0.1844940185546875, "val/ratio": 0.9872738122940063, "val/ratio_var": 3.0919560231268406e-05, "val/num_eos_tokens": 362, "lr": 1.6882338204164212e-06, "episode": 14752, "epoch": 0.7450505050505051, "step": 922}, {"eps": 0, "objective/kl": 40.59729766845703, "objective/entropy": -15.746962547302246, "objective/non_score_reward": -2.029865026473999, "objective/rlhf_reward": 12.388103485107422, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0010770086664706469, "policy/clipfrac_avg": 0.005413630045950413, "loss/policy_avg": -0.07059650868177414, "loss/value_avg": 0.6463966369628906, "val/clipfrac_avg": 0.00856552179902792, "policy/entropy_avg": 0.21207809448242188, "val/ratio": 0.9861268401145935, "val/ratio_var": 5.63512985536363e-05, "val/num_eos_tokens": 175, "lr": 1.67823861246477e-06, "episode": 14768, "epoch": 0.7458585858585859, "step": 923}, {"eps": 0, "objective/kl": 34.25519943237305, "objective/entropy": -118.91422271728516, "objective/non_score_reward": -1.7127599716186523, "objective/rlhf_reward": 12.224739074707031, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0007098158821463585, "policy/clipfrac_avg": 0.0022262982092797756, "loss/policy_avg": 0.06915351748466492, "loss/value_avg": 1.4651566743850708, "val/clipfrac_avg": 0.06050518900156021, "policy/entropy_avg": 0.1386256217956543, "val/ratio": 0.9912421703338623, "val/ratio_var": 4.530951628112234e-05, "val/num_eos_tokens": 270, "lr": 1.6682671102399806e-06, "episode": 14784, "epoch": 0.7466666666666667, "step": 924}, {"eps": 0, "objective/kl": 38.95854949951172, "objective/entropy": -81.5009765625, "objective/non_score_reward": -1.947927474975586, "objective/rlhf_reward": 15.450510025024414, "objective/scores": 17.375, "policy/approxkl_avg": 0.0009277502540498972, "policy/clipfrac_avg": 0.004952313844114542, "loss/policy_avg": 0.18248242139816284, "loss/value_avg": 1.6025516986846924, "val/clipfrac_avg": 0.016582708805799484, "policy/entropy_avg": 0.16534215211868286, "val/ratio": 0.9883300065994263, "val/ratio_var": 5.890153624932282e-05, "val/num_eos_tokens": 323, "lr": 1.6583193849036095e-06, "episode": 14800, "epoch": 0.7474747474747475, "step": 925}, {"eps": 0, "objective/kl": 35.18474578857422, "objective/entropy": -46.540870666503906, "objective/non_score_reward": -1.75923752784729, "objective/rlhf_reward": 13.225137710571289, "objective/scores": 15.0, "policy/approxkl_avg": 0.0009727986180223525, "policy/clipfrac_avg": 0.0023795529268682003, "loss/policy_avg": -0.16795751452445984, "loss/value_avg": 0.3992899954319, "val/clipfrac_avg": 0.0444885715842247, "policy/entropy_avg": 0.21690034866333008, "val/ratio": 0.9869061708450317, "val/ratio_var": 6.481046148110181e-05, "val/num_eos_tokens": 163, "lr": 1.6483955074475278e-06, "episode": 14816, "epoch": 0.7482828282828283, "step": 926}, {"eps": 0, "objective/kl": 33.047142028808594, "objective/entropy": -73.27378845214844, "objective/non_score_reward": -1.6523573398590088, "objective/rlhf_reward": 13.33201789855957, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008205980993807316, "policy/clipfrac_avg": 0.0039194016717374325, "loss/policy_avg": -0.3186694383621216, "loss/value_avg": 0.933090090751648, "val/clipfrac_avg": 0.07463380694389343, "policy/entropy_avg": 0.17404580116271973, "val/ratio": 0.9890973567962646, "val/ratio_var": 8.332987636094913e-05, "val/num_eos_tokens": 165, "lr": 1.6384955486934157e-06, "episode": 14832, "epoch": 0.7490909090909091, "step": 927}, {"eps": 0, "objective/kl": 41.2000846862793, "objective/entropy": -108.95191192626953, "objective/non_score_reward": -2.060004472732544, "objective/rlhf_reward": 14.135308265686035, "objective/scores": 16.25, "policy/approxkl_avg": 0.0007390918326564133, "policy/clipfrac_avg": 0.0021800659596920013, "loss/policy_avg": -0.018651030957698822, "loss/value_avg": 0.23730480670928955, "val/clipfrac_avg": 0.02866453304886818, "policy/entropy_avg": 0.15365886688232422, "val/ratio": 0.989920973777771, "val/ratio_var": 5.118203625897877e-05, "val/num_eos_tokens": 497, "lr": 1.6286195792922655e-06, "episode": 14848, "epoch": 0.74989898989899, "step": 928}, {"eps": 0, "objective/kl": 47.88815689086914, "objective/entropy": -53.78815841674805, "objective/non_score_reward": -2.3944079875946045, "objective/rlhf_reward": 12.574342727661133, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008938192040659487, "policy/clipfrac_avg": 0.003149396274238825, "loss/policy_avg": 0.11356082558631897, "loss/value_avg": 0.5069226622581482, "val/clipfrac_avg": 0.0172991082072258, "policy/entropy_avg": 0.181840181350708, "val/ratio": 0.9889012575149536, "val/ratio_var": 3.912192551069893e-05, "val/num_eos_tokens": 19, "lr": 1.6187676697238564e-06, "episode": 14864, "epoch": 0.7507070707070707, "step": 929}, {"eps": 0, "objective/kl": 37.49846649169922, "objective/entropy": -104.1807861328125, "objective/non_score_reward": -1.8749234676361084, "objective/rlhf_reward": 15.953201293945312, "objective/scores": 17.875, "policy/approxkl_avg": 0.0009115774882957339, "policy/clipfrac_avg": 0.005451949778944254, "loss/policy_avg": -0.07865975797176361, "loss/value_avg": 0.41433531045913696, "val/clipfrac_avg": 0.058040205389261246, "policy/entropy_avg": 0.1745290756225586, "val/ratio": 0.9887331128120422, "val/ratio_var": 6.705939449602738e-05, "val/num_eos_tokens": 27, "lr": 1.6089398902962767e-06, "episode": 14880, "epoch": 0.7515151515151515, "step": 930}, {"eps": 0, "objective/kl": 42.68102264404297, "objective/entropy": -97.50758361816406, "objective/non_score_reward": -2.1340510845184326, "objective/rlhf_reward": 13.498762130737305, "objective/scores": 15.625, "policy/approxkl_avg": 0.0008075715741142631, "policy/clipfrac_avg": 0.011311197653412819, "loss/policy_avg": -0.035402484238147736, "loss/value_avg": 0.5358971357345581, "val/clipfrac_avg": 0.003885046113282442, "policy/entropy_avg": 0.1630105972290039, "val/ratio": 0.9890309572219849, "val/ratio_var": 5.2104482165304944e-05, "val/num_eos_tokens": 302, "lr": 1.5991363111454023e-06, "episode": 14896, "epoch": 0.7523232323232323, "step": 931}, {"eps": 0, "objective/kl": 32.467742919921875, "objective/entropy": -78.21574401855469, "objective/non_score_reward": -1.623387098312378, "objective/rlhf_reward": 14.372706413269043, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008647768991068006, "policy/clipfrac_avg": 0.005656212102621794, "loss/policy_avg": -0.30045461654663086, "loss/value_avg": 0.872713565826416, "val/clipfrac_avg": 0.0015919646248221397, "policy/entropy_avg": 0.17721939086914062, "val/ratio": 0.9885766506195068, "val/ratio_var": 6.424664752557874e-05, "val/num_eos_tokens": 12, "lr": 1.5893570022344124e-06, "episode": 14912, "epoch": 0.7531313131313131, "step": 932}, {"eps": 0, "objective/kl": 36.504150390625, "objective/entropy": -61.095802307128906, "objective/non_score_reward": -1.8252074718475342, "objective/rlhf_reward": 12.393542289733887, "objective/scores": 14.25, "policy/approxkl_avg": 0.0008130621863529086, "policy/clipfrac_avg": 0.005108194425702095, "loss/policy_avg": -0.012097490951418877, "loss/value_avg": 0.7004050016403198, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16521072387695312, "val/ratio": 0.9887269735336304, "val/ratio_var": 6.136602314654738e-05, "val/num_eos_tokens": 182, "lr": 1.5796020333532696e-06, "episode": 14928, "epoch": 0.7539393939393939, "step": 933}, {"eps": 0, "objective/kl": 34.973388671875, "objective/entropy": -110.07270812988281, "objective/non_score_reward": -1.7486693859100342, "objective/rlhf_reward": 14.548205375671387, "objective/scores": 16.25, "policy/approxkl_avg": 0.00091616774443537, "policy/clipfrac_avg": 0.006772395223379135, "loss/policy_avg": 0.03149443119764328, "loss/value_avg": 0.3308320641517639, "val/clipfrac_avg": 0.0015980113530531526, "policy/entropy_avg": 0.17972993850708008, "val/ratio": 0.9878275394439697, "val/ratio_var": 8.512718341080472e-05, "val/num_eos_tokens": 350, "lr": 1.5698714741182459e-06, "episode": 14944, "epoch": 0.7547474747474747, "step": 934}, {"eps": 0, "objective/kl": 39.53004837036133, "objective/entropy": -25.034250259399414, "objective/non_score_reward": -1.976502537727356, "objective/rlhf_reward": 14.281310081481934, "objective/scores": 16.25, "policy/approxkl_avg": 0.00102006527595222, "policy/clipfrac_avg": 0.00497299712151289, "loss/policy_avg": -0.03353989124298096, "loss/value_avg": 0.4103987216949463, "val/clipfrac_avg": 0.0242628026753664, "policy/entropy_avg": 0.19387245178222656, "val/ratio": 0.9875220060348511, "val/ratio_var": 6.656184268649668e-05, "val/num_eos_tokens": 73, "lr": 1.5601653939714073e-06, "episode": 14960, "epoch": 0.7555555555555555, "step": 935}, {"eps": 0, "objective/kl": 37.98503494262695, "objective/entropy": -60.77607345581055, "objective/non_score_reward": -1.8992516994476318, "objective/rlhf_reward": 13.171060562133789, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0007948282873257995, "policy/clipfrac_avg": 0.002496684668585658, "loss/policy_avg": -0.1474141776561737, "loss/value_avg": 0.527273416519165, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1510624885559082, "val/ratio": 0.9904986619949341, "val/ratio_var": 3.4012387914117426e-05, "val/num_eos_tokens": 96, "lr": 1.5504838621801272e-06, "episode": 14976, "epoch": 0.7563636363636363, "step": 936}, {"eps": 0, "objective/kl": 41.95027160644531, "objective/entropy": -60.082557678222656, "objective/non_score_reward": -2.0975136756896973, "objective/rlhf_reward": 12.652486801147461, "objective/scores": 14.75, "policy/approxkl_avg": 0.0009200765052810311, "policy/clipfrac_avg": 0.0059701185673475266, "loss/policy_avg": 0.1368754655122757, "loss/value_avg": 0.6238852143287659, "val/clipfrac_avg": 0.012415202334523201, "policy/entropy_avg": 0.17743873596191406, "val/ratio": 0.9879815578460693, "val/ratio_var": 4.387104490888305e-05, "val/num_eos_tokens": 54, "lr": 1.5408269478365883e-06, "episode": 14992, "epoch": 0.7571717171717172, "step": 937}, {"eps": 0, "objective/kl": 35.57097625732422, "objective/entropy": -112.668701171875, "objective/non_score_reward": -1.7785489559173584, "objective/rlhf_reward": 13.416763305664062, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0008795931353233755, "policy/clipfrac_avg": 0.005144854076206684, "loss/policy_avg": -0.17327642440795898, "loss/value_avg": 0.538246750831604, "val/clipfrac_avg": 0.047972679138183594, "policy/entropy_avg": 0.16583538055419922, "val/ratio": 0.9902178645133972, "val/ratio_var": 5.2144809160381556e-05, "val/num_eos_tokens": 136, "lr": 1.5311947198572918e-06, "episode": 15008, "epoch": 0.757979797979798, "step": 938}, {"eps": 0, "objective/kl": 35.88062286376953, "objective/entropy": -72.79998779296875, "objective/non_score_reward": -1.7940311431884766, "objective/rlhf_reward": 14.46378231048584, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008520716801285744, "policy/clipfrac_avg": 0.0030313697643578053, "loss/policy_avg": 0.5920745134353638, "loss/value_avg": 1.1476694345474243, "val/clipfrac_avg": 0.021162573248147964, "policy/entropy_avg": 0.19948351383209229, "val/ratio": 0.9882235527038574, "val/ratio_var": 6.201990618137643e-05, "val/num_eos_tokens": 210, "lr": 1.5215872469825682e-06, "episode": 15024, "epoch": 0.7587878787878788, "step": 939}, {"eps": 0, "objective/kl": 42.35734176635742, "objective/entropy": -91.86679077148438, "objective/non_score_reward": -2.1178669929504395, "objective/rlhf_reward": 15.264945983886719, "objective/scores": 17.375, "policy/approxkl_avg": 0.0010741858277469873, "policy/clipfrac_avg": 0.00790454912930727, "loss/policy_avg": -0.15639176964759827, "loss/value_avg": 0.8813156485557556, "val/clipfrac_avg": 0.015625, "policy/entropy_avg": 0.19293498992919922, "val/ratio": 0.986987829208374, "val/ratio_var": 6.426944310078397e-05, "val/num_eos_tokens": 9, "lr": 1.5120045977760784e-06, "episode": 15040, "epoch": 0.7595959595959596, "step": 940}, {"eps": 0, "objective/kl": 31.252155303955078, "objective/entropy": -62.31670379638672, "objective/non_score_reward": -1.562608003616333, "objective/rlhf_reward": 14.156142234802246, "objective/scores": 15.75, "policy/approxkl_avg": 0.0007184218848124146, "policy/clipfrac_avg": 0.0035393601283431053, "loss/policy_avg": 0.2165047824382782, "loss/value_avg": 0.34554335474967957, "val/clipfrac_avg": 0.013209432363510132, "policy/entropy_avg": 0.14359378814697266, "val/ratio": 0.9896091222763062, "val/ratio_var": 5.599396536126733e-05, "val/num_eos_tokens": 36, "lr": 1.502446840624331e-06, "episode": 15056, "epoch": 0.7604040404040404, "step": 941}, {"eps": 0, "objective/kl": 38.075103759765625, "objective/entropy": -109.5699691772461, "objective/non_score_reward": -1.9037551879882812, "objective/rlhf_reward": 14.713432312011719, "objective/scores": 16.625, "policy/approxkl_avg": 0.0010412151459604502, "policy/clipfrac_avg": 0.009930627420544624, "loss/policy_avg": -0.030075984075665474, "loss/value_avg": 0.45760178565979004, "val/clipfrac_avg": 0.014139890670776367, "policy/entropy_avg": 0.18667316436767578, "val/ratio": 0.9863598346710205, "val/ratio_var": 0.00010634912905516103, "val/num_eos_tokens": 105, "lr": 1.4929140437361916e-06, "episode": 15072, "epoch": 0.7612121212121212, "step": 942}, {"eps": 0, "objective/kl": 34.23678207397461, "objective/entropy": -121.374267578125, "objective/non_score_reward": -1.711839199066162, "objective/rlhf_reward": 14.038161277770996, "objective/scores": 15.75, "policy/approxkl_avg": 0.0007155380444601178, "policy/clipfrac_avg": 0.00446352269500494, "loss/policy_avg": 0.12722979485988617, "loss/value_avg": 0.5696362853050232, "val/clipfrac_avg": 0.001972501166164875, "policy/entropy_avg": 0.13922500610351562, "val/ratio": 0.9906710386276245, "val/ratio_var": 6.088761438149959e-05, "val/num_eos_tokens": 493, "lr": 1.4834062751424018e-06, "episode": 15088, "epoch": 0.762020202020202, "step": 943}, {"eps": 0, "objective/kl": 42.60675811767578, "objective/entropy": -117.67542266845703, "objective/non_score_reward": -2.130337953567505, "objective/rlhf_reward": 13.854036331176758, "objective/scores": 16.0, "policy/approxkl_avg": 0.0007362811593338847, "policy/clipfrac_avg": 0.009282913990318775, "loss/policy_avg": 0.24459479749202728, "loss/value_avg": 0.997427225112915, "val/clipfrac_avg": 0.019384628161787987, "policy/entropy_avg": 0.15163874626159668, "val/ratio": 0.9907106161117554, "val/ratio_var": 5.1745548262260854e-05, "val/num_eos_tokens": 404, "lr": 1.4739236026950838e-06, "episode": 15104, "epoch": 0.7628282828282829, "step": 944}, {"eps": 0, "objective/kl": 36.86358642578125, "objective/entropy": -72.49942779541016, "objective/non_score_reward": -1.8431792259216309, "objective/rlhf_reward": 14.227133750915527, "objective/scores": 16.125, "policy/approxkl_avg": 0.0009655736503191292, "policy/clipfrac_avg": 0.0035610971972346306, "loss/policy_avg": 0.019024059176445007, "loss/value_avg": 0.2505372166633606, "val/clipfrac_avg": 0.0008617835119366646, "policy/entropy_avg": 0.19275665283203125, "val/ratio": 0.9873601198196411, "val/ratio_var": 7.99532572273165e-05, "val/num_eos_tokens": 33, "lr": 1.4644660940672628e-06, "episode": 15120, "epoch": 0.7636363636363637, "step": 945}, {"eps": 0, "objective/kl": 30.419466018676758, "objective/entropy": -106.38670349121094, "objective/non_score_reward": -1.5209734439849854, "objective/rlhf_reward": 15.338401794433594, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009057885035872459, "policy/clipfrac_avg": 0.009060706943273544, "loss/policy_avg": -0.24745053052902222, "loss/value_avg": 0.4052670896053314, "val/clipfrac_avg": 0.038856737315654755, "policy/entropy_avg": 0.161224365234375, "val/ratio": 0.9891325235366821, "val/ratio_var": 9.022479935083538e-05, "val/num_eos_tokens": 409, "lr": 1.455033816752383e-06, "episode": 15136, "epoch": 0.7644444444444445, "step": 946}, {"eps": 0, "objective/kl": 36.21892547607422, "objective/entropy": -104.14939880371094, "objective/non_score_reward": -1.8109462261199951, "objective/rlhf_reward": 15.282804489135742, "objective/scores": 17.125, "policy/approxkl_avg": 0.0009193423902615905, "policy/clipfrac_avg": 0.010075472295284271, "loss/policy_avg": -0.051016077399253845, "loss/value_avg": 0.5337303876876831, "val/clipfrac_avg": 0.006654549390077591, "policy/entropy_avg": 0.16665363311767578, "val/ratio": 0.9888213276863098, "val/ratio_var": 8.222387987188995e-05, "val/num_eos_tokens": 424, "lr": 1.4456268380638262e-06, "episode": 15152, "epoch": 0.7652525252525253, "step": 947}, {"eps": 0, "objective/kl": 37.6370849609375, "objective/entropy": -107.8341064453125, "objective/non_score_reward": -1.8818542957305908, "objective/rlhf_reward": 11.461895942687988, "objective/scores": 13.375, "policy/approxkl_avg": 0.0008800949435681105, "policy/clipfrac_avg": 0.018943525850772858, "loss/policy_avg": 0.5708428025245667, "loss/value_avg": 0.9156904220581055, "val/clipfrac_avg": 0.0006793478387407959, "policy/entropy_avg": 0.18418407440185547, "val/ratio": 0.9880671501159668, "val/ratio_var": 7.767622446408495e-05, "val/num_eos_tokens": 297, "lr": 1.4362452251344283e-06, "episode": 15168, "epoch": 0.7660606060606061, "step": 948}, {"eps": 0, "objective/kl": 34.966495513916016, "objective/entropy": -11.566868782043457, "objective/non_score_reward": -1.7483247518539429, "objective/rlhf_reward": 14.603238105773926, "objective/scores": 16.375, "policy/approxkl_avg": 0.0011735728476196527, "policy/clipfrac_avg": 0.004941962193697691, "loss/policy_avg": -0.028068287298083305, "loss/value_avg": 0.6219231486320496, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20639419555664062, "val/ratio": 0.9847333431243896, "val/ratio_var": 3.736758299055509e-05, "val/num_eos_tokens": 32, "lr": 1.4268890449160034e-06, "episode": 15184, "epoch": 0.7668686868686869, "step": 949}, {"eps": 0, "objective/kl": 31.678754806518555, "objective/entropy": -80.4765625, "objective/non_score_reward": -1.583937644958496, "objective/rlhf_reward": 13.853562355041504, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0006830325582996011, "policy/clipfrac_avg": 0.00417131744325161, "loss/policy_avg": -0.15076535940170288, "loss/value_avg": 0.2775721549987793, "val/clipfrac_avg": 0.0393822118639946, "policy/entropy_avg": 0.1293621063232422, "val/ratio": 0.9911376237869263, "val/ratio_var": 2.9909415388829075e-05, "val/num_eos_tokens": 257, "lr": 1.4175583641788698e-06, "episode": 15200, "epoch": 0.7676767676767676, "step": 950}, {"eps": 0, "objective/kl": 38.69972229003906, "objective/entropy": -84.18096160888672, "objective/non_score_reward": -1.9349862337112427, "objective/rlhf_reward": 15.416576385498047, "objective/scores": 17.375, "policy/approxkl_avg": 0.000798642635345459, "policy/clipfrac_avg": 0.0038171219639480114, "loss/policy_avg": -0.04640840366482735, "loss/value_avg": 0.3922756314277649, "val/clipfrac_avg": 0.0032731681130826473, "policy/entropy_avg": 0.1776995062828064, "val/ratio": 0.9889781475067139, "val/ratio_var": 5.715083170798607e-05, "val/num_eos_tokens": 326, "lr": 1.4082532495113627e-06, "episode": 15216, "epoch": 0.7684848484848484, "step": 951}, {"eps": 0, "objective/kl": 39.91594314575195, "objective/entropy": -60.43904113769531, "objective/non_score_reward": -1.9957971572875977, "objective/rlhf_reward": 14.262016296386719, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009779632091522217, "policy/clipfrac_avg": 0.004585625603795052, "loss/policy_avg": -0.12606917321681976, "loss/value_avg": 0.3175104856491089, "val/clipfrac_avg": 0.030662208795547485, "policy/entropy_avg": 0.2008209228515625, "val/ratio": 0.9862244129180908, "val/ratio_var": 8.592133963247761e-05, "val/num_eos_tokens": 422, "lr": 1.3989737673193682e-06, "episode": 15232, "epoch": 0.7692929292929293, "step": 952}, {"eps": 0, "objective/kl": 46.73610305786133, "objective/entropy": -67.14826965332031, "objective/non_score_reward": -2.3368051052093506, "objective/rlhf_reward": 14.29600715637207, "objective/scores": 16.625, "policy/approxkl_avg": 0.0011253968114033341, "policy/clipfrac_avg": 0.006955197546631098, "loss/policy_avg": 0.08768834173679352, "loss/value_avg": 0.327705055475235, "val/clipfrac_avg": 8.926542068365961e-05, "policy/entropy_avg": 0.20795369148254395, "val/ratio": 0.9854748249053955, "val/ratio_var": 6.634320016019046e-05, "val/num_eos_tokens": 421, "lr": 1.389719983825844e-06, "episode": 15248, "epoch": 0.7701010101010101, "step": 953}, {"eps": 0, "objective/kl": 39.47554016113281, "objective/entropy": -52.92948913574219, "objective/non_score_reward": -1.9737770557403564, "objective/rlhf_reward": 15.323097229003906, "objective/scores": 17.25, "policy/approxkl_avg": 0.0009001094149425626, "policy/clipfrac_avg": 0.005291195120662451, "loss/policy_avg": 0.17172515392303467, "loss/value_avg": 0.5182530283927917, "val/clipfrac_avg": 0.006615057587623596, "policy/entropy_avg": 0.18181610107421875, "val/ratio": 0.9876183271408081, "val/ratio_var": 5.796940240543336e-05, "val/num_eos_tokens": 490, "lr": 1.3804919650703551e-06, "episode": 15264, "epoch": 0.7709090909090909, "step": 954}, {"eps": 0, "objective/kl": 41.423980712890625, "objective/entropy": -93.25178527832031, "objective/non_score_reward": -2.0711989402770996, "objective/rlhf_reward": 13.155363082885742, "objective/scores": 15.25, "policy/approxkl_avg": 0.0009469924843870103, "policy/clipfrac_avg": 0.004164956044405699, "loss/policy_avg": 0.3471048176288605, "loss/value_avg": 0.7186154723167419, "val/clipfrac_avg": 0.04515676945447922, "policy/entropy_avg": 0.18822860717773438, "val/ratio": 0.9877511262893677, "val/ratio_var": 7.300172001123428e-05, "val/num_eos_tokens": 611, "lr": 1.3712897769085903e-06, "episode": 15280, "epoch": 0.7717171717171717, "step": 955}, {"eps": 0, "objective/kl": 31.20224380493164, "objective/entropy": -112.50715637207031, "objective/non_score_reward": -1.5601121187210083, "objective/rlhf_reward": 13.799263000488281, "objective/scores": 15.375, "policy/approxkl_avg": 0.0007952903397381306, "policy/clipfrac_avg": 0.008523119613528252, "loss/policy_avg": 0.24007092416286469, "loss/value_avg": 0.4516885280609131, "val/clipfrac_avg": 0.06015515699982643, "policy/entropy_avg": 0.1457352638244629, "val/ratio": 0.9895913600921631, "val/ratio_var": 7.848602399462834e-05, "val/num_eos_tokens": 383, "lr": 1.3621134850119012e-06, "episode": 15296, "epoch": 0.7725252525252525, "step": 956}, {"eps": 0, "objective/kl": 44.714515686035156, "objective/entropy": -82.6330337524414, "objective/non_score_reward": -2.2357258796691895, "objective/rlhf_reward": 13.818962097167969, "objective/scores": 16.0, "policy/approxkl_avg": 0.0006799673428758979, "policy/clipfrac_avg": 0.003063515294343233, "loss/policy_avg": -0.03721753507852554, "loss/value_avg": 0.241817444562912, "val/clipfrac_avg": 9.245562250725925e-05, "policy/entropy_avg": 0.1481151580810547, "val/ratio": 0.9905509948730469, "val/ratio_var": 4.010691918665543e-05, "val/num_eos_tokens": 341, "lr": 1.3529631548668298e-06, "episode": 15312, "epoch": 0.7733333333333333, "step": 957}, {"eps": 0, "objective/kl": 32.26376724243164, "objective/entropy": -75.73651885986328, "objective/non_score_reward": -1.6131882667541504, "objective/rlhf_reward": 13.550874710083008, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0010009508114308119, "policy/clipfrac_avg": 0.004912985023111105, "loss/policy_avg": -0.1448572874069214, "loss/value_avg": 0.6361722946166992, "val/clipfrac_avg": 0.008336715400218964, "policy/entropy_avg": 0.190643310546875, "val/ratio": 0.9864659905433655, "val/ratio_var": 9.685769327916205e-05, "val/num_eos_tokens": 329, "lr": 1.3438388517746431e-06, "episode": 15328, "epoch": 0.7741414141414141, "step": 958}, {"eps": 0, "objective/kl": 39.02738952636719, "objective/entropy": -90.25643920898438, "objective/non_score_reward": -1.9513695240020752, "objective/rlhf_reward": 14.954880714416504, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009018229902721941, "policy/clipfrac_avg": 0.004778256639838219, "loss/policy_avg": 0.11857907474040985, "loss/value_avg": 0.8868049383163452, "val/clipfrac_avg": 0.02706507220864296, "policy/entropy_avg": 0.1796565055847168, "val/ratio": 0.9882634878158569, "val/ratio_var": 9.323142148787156e-05, "val/num_eos_tokens": 289, "lr": 1.3347406408508695e-06, "episode": 15344, "epoch": 0.7749494949494949, "step": 959}, {"eps": 0, "objective/kl": 37.34764099121094, "objective/entropy": -101.29171752929688, "objective/non_score_reward": -1.8673819303512573, "objective/rlhf_reward": 14.32011890411377, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008543966105207801, "policy/clipfrac_avg": 0.0047050644643604755, "loss/policy_avg": -0.22503328323364258, "loss/value_avg": 0.4810031056404114, "val/clipfrac_avg": 0.0005405173869803548, "policy/entropy_avg": 0.17150402069091797, "val/ratio": 0.9882644414901733, "val/ratio_var": 6.539086461998522e-05, "val/num_eos_tokens": 130, "lr": 1.3256685870248227e-06, "episode": 15360, "epoch": 0.7757575757575758, "step": 960}, {"eps": 0, "objective/kl": 42.212120056152344, "objective/entropy": -97.00421142578125, "objective/non_score_reward": -2.1106061935424805, "objective/rlhf_reward": 13.99876880645752, "objective/scores": 16.125, "policy/approxkl_avg": 0.000946523156017065, "policy/clipfrac_avg": 0.0059087686240673065, "loss/policy_avg": -0.0476013720035553, "loss/value_avg": 0.682388961315155, "val/clipfrac_avg": 0.0003124999930150807, "policy/entropy_avg": 0.19147014617919922, "val/ratio": 0.9874411225318909, "val/ratio_var": 5.3472565923584625e-05, "val/num_eos_tokens": 224, "lr": 1.3166227550391542e-06, "episode": 15376, "epoch": 0.7765656565656566, "step": 961}, {"eps": 0, "objective/kl": 39.21135330200195, "objective/entropy": -60.28771209716797, "objective/non_score_reward": -1.9605677127838135, "objective/rlhf_reward": 15.351932525634766, "objective/scores": 17.25, "policy/approxkl_avg": 0.0009600625489838421, "policy/clipfrac_avg": 0.0035959878005087376, "loss/policy_avg": -0.26426494121551514, "loss/value_avg": 0.47233304381370544, "val/clipfrac_avg": 0.012040323577821255, "policy/entropy_avg": 0.18239498138427734, "val/ratio": 0.9875339269638062, "val/ratio_var": 6.485576159320772e-05, "val/num_eos_tokens": 296, "lr": 1.307603209449378e-06, "episode": 15392, "epoch": 0.7773737373737374, "step": 962}, {"eps": 0, "objective/kl": 38.37220001220703, "objective/entropy": -146.63661193847656, "objective/non_score_reward": -1.9186100959777832, "objective/rlhf_reward": 13.354827880859375, "objective/scores": 15.25, "policy/approxkl_avg": 0.0007201317930594087, "policy/clipfrac_avg": 0.0022899312898516655, "loss/policy_avg": 0.04519583284854889, "loss/value_avg": 0.7595553398132324, "val/clipfrac_avg": 8.292270649690181e-05, "policy/entropy_avg": 0.1562885046005249, "val/ratio": 0.9906989336013794, "val/ratio_var": 6.185618985909969e-05, "val/num_eos_tokens": 521, "lr": 1.298610014623423e-06, "episode": 15408, "epoch": 0.7781818181818182, "step": 963}, {"eps": 0, "objective/kl": 36.48185348510742, "objective/entropy": -66.08819580078125, "objective/non_score_reward": -1.8240927457809448, "objective/rlhf_reward": 14.261844635009766, "objective/scores": 16.125, "policy/approxkl_avg": 0.0009832159848883748, "policy/clipfrac_avg": 0.006735666189342737, "loss/policy_avg": -0.14623457193374634, "loss/value_avg": 0.35101252794265747, "val/clipfrac_avg": 0.0030241934582591057, "policy/entropy_avg": 0.19070911407470703, "val/ratio": 0.9872285723686218, "val/ratio_var": 7.909470878075808e-05, "val/num_eos_tokens": 421, "lr": 1.2896432347411553e-06, "episode": 15424, "epoch": 0.778989898989899, "step": 964}, {"eps": 0, "objective/kl": 36.51701354980469, "objective/entropy": -48.76942825317383, "objective/non_score_reward": -1.8258507251739502, "objective/rlhf_reward": 13.861648559570312, "objective/scores": 15.6875, "policy/approxkl_avg": 0.000836203049402684, "policy/clipfrac_avg": 0.006686762906610966, "loss/policy_avg": -0.1669466346502304, "loss/value_avg": 0.6373587846755981, "val/clipfrac_avg": 0.009943181648850441, "policy/entropy_avg": 0.17627906799316406, "val/ratio": 0.9892607927322388, "val/ratio_var": 5.405569390859455e-05, "val/num_eos_tokens": 11, "lr": 1.2807029337939403e-06, "episode": 15440, "epoch": 0.7797979797979798, "step": 965}, {"eps": 0, "objective/kl": 42.5623893737793, "objective/entropy": -59.34945297241211, "objective/non_score_reward": -2.128119468688965, "objective/rlhf_reward": 13.918755531311035, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008710323600098491, "policy/clipfrac_avg": 0.010207775980234146, "loss/policy_avg": -0.005087921395897865, "loss/value_avg": 0.2712212800979614, "val/clipfrac_avg": 0.010802468284964561, "policy/entropy_avg": 0.1739978790283203, "val/ratio": 0.9884668588638306, "val/ratio_var": 4.567186624626629e-05, "val/num_eos_tokens": 246, "lr": 1.2717891755841722e-06, "episode": 15456, "epoch": 0.7806060606060606, "step": 966}, {"eps": 0, "objective/kl": 29.273542404174805, "objective/entropy": -136.59652709960938, "objective/non_score_reward": -1.463677167892456, "objective/rlhf_reward": 14.083197593688965, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0007826394285075366, "policy/clipfrac_avg": 0.006299240980297327, "loss/policy_avg": -0.5036620497703552, "loss/value_avg": 0.39513152837753296, "val/clipfrac_avg": 0.03271849453449249, "policy/entropy_avg": 0.15417519211769104, "val/ratio": 0.9902065992355347, "val/ratio_var": 8.734448783798143e-05, "val/num_eos_tokens": 249, "lr": 1.2629020237248241e-06, "episode": 15472, "epoch": 0.7814141414141414, "step": 967}, {"eps": 0, "objective/kl": 38.655548095703125, "objective/entropy": -29.31902503967285, "objective/non_score_reward": -1.9327776432037354, "objective/rlhf_reward": 13.864096641540527, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0008980755810625851, "policy/clipfrac_avg": 0.003656789194792509, "loss/policy_avg": -0.04834941402077675, "loss/value_avg": 0.49886178970336914, "val/clipfrac_avg": 5.3879310144111514e-05, "policy/entropy_avg": 0.1847667694091797, "val/ratio": 0.9879658818244934, "val/ratio_var": 2.3956074073794298e-05, "val/num_eos_tokens": 10, "lr": 1.2540415416389917e-06, "episode": 15488, "epoch": 0.7822222222222223, "step": 968}, {"eps": 0, "objective/kl": 41.82144546508789, "objective/entropy": -206.95504760742188, "objective/non_score_reward": -2.0910723209381104, "objective/rlhf_reward": 13.076896667480469, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0008509804611094296, "policy/clipfrac_avg": 0.0047798133455216885, "loss/policy_avg": -0.2217503786087036, "loss/value_avg": 0.4100498557090759, "val/clipfrac_avg": 0.00021062050655018538, "policy/entropy_avg": 0.15673446655273438, "val/ratio": 0.9887630939483643, "val/ratio_var": 6.604092777706683e-05, "val/num_eos_tokens": 481, "lr": 1.2452077925594435e-06, "episode": 15504, "epoch": 0.7830303030303031, "step": 969}, {"eps": 0, "objective/kl": 38.07307052612305, "objective/entropy": -40.900238037109375, "objective/non_score_reward": -1.9036535024642944, "objective/rlhf_reward": 14.377596855163574, "objective/scores": 16.25, "policy/approxkl_avg": 0.0010727059561759233, "policy/clipfrac_avg": 0.004337977152317762, "loss/policy_avg": -0.02120068296790123, "loss/value_avg": 0.6901993751525879, "val/clipfrac_avg": 0.02858383022248745, "policy/entropy_avg": 0.1984100341796875, "val/ratio": 0.9863028526306152, "val/ratio_var": 8.229557715822011e-05, "val/num_eos_tokens": 535, "lr": 1.23640083952817e-06, "episode": 15520, "epoch": 0.7838383838383839, "step": 970}, {"eps": 0, "objective/kl": 39.35569381713867, "objective/entropy": -77.6534652709961, "objective/non_score_reward": -1.9677847623825073, "objective/rlhf_reward": 13.907215118408203, "objective/scores": 15.875, "policy/approxkl_avg": 0.0008703683270141482, "policy/clipfrac_avg": 0.0043828776106238365, "loss/policy_avg": -0.09917282313108444, "loss/value_avg": 0.3991038203239441, "val/clipfrac_avg": 0.00012400794366840273, "policy/entropy_avg": 0.17210960388183594, "val/ratio": 0.988074541091919, "val/ratio_var": 5.318454350344837e-05, "val/num_eos_tokens": 175, "lr": 1.2276207453959283e-06, "episode": 15536, "epoch": 0.7846464646464646, "step": 971}, {"eps": 0, "objective/kl": 41.14687728881836, "objective/entropy": -16.92827606201172, "objective/non_score_reward": -2.0573439598083496, "objective/rlhf_reward": 15.442655563354492, "objective/scores": 17.5, "policy/approxkl_avg": 0.0010900306515395641, "policy/clipfrac_avg": 0.00866631604731083, "loss/policy_avg": 0.0814996063709259, "loss/value_avg": 0.6175538897514343, "val/clipfrac_avg": 0.009025804698467255, "policy/entropy_avg": 0.2063446044921875, "val/ratio": 0.9854716062545776, "val/ratio_var": 2.7480951757752337e-05, "val/num_eos_tokens": 174, "lr": 1.2188675728217986e-06, "episode": 15552, "epoch": 0.7854545454545454, "step": 972}, {"eps": 0, "objective/kl": 45.04676818847656, "objective/entropy": -63.132266998291016, "objective/non_score_reward": -2.252338409423828, "objective/rlhf_reward": 14.450786590576172, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009616734459996223, "policy/clipfrac_avg": 0.0046102688647806644, "loss/policy_avg": 0.3593114912509918, "loss/value_avg": 0.7372395396232605, "val/clipfrac_avg": 0.016889449208974838, "policy/entropy_avg": 0.1856832504272461, "val/ratio": 0.9877485036849976, "val/ratio_var": 6.443963502533734e-05, "val/num_eos_tokens": 419, "lr": 1.2101413842727345e-06, "episode": 15568, "epoch": 0.7862626262626262, "step": 973}, {"eps": 0, "objective/kl": 41.9028434753418, "objective/entropy": -98.38099670410156, "objective/non_score_reward": -2.095142126083374, "objective/rlhf_reward": 12.186107635498047, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009328118758276105, "policy/clipfrac_avg": 0.004906289279460907, "loss/policy_avg": 0.14888805150985718, "loss/value_avg": 0.7544107437133789, "val/clipfrac_avg": 0.08914318680763245, "policy/entropy_avg": 0.17488479614257812, "val/ratio": 0.9888035655021667, "val/ratio_var": 6.763465353287756e-05, "val/num_eos_tokens": 193, "lr": 1.201442242023122e-06, "episode": 15584, "epoch": 0.787070707070707, "step": 974}, {"eps": 0, "objective/kl": 33.80537796020508, "objective/entropy": -65.54731750488281, "objective/non_score_reward": -1.6902689933776855, "objective/rlhf_reward": 13.434730529785156, "objective/scores": 15.125, "policy/approxkl_avg": 0.0010051055578514934, "policy/clipfrac_avg": 0.005396673921495676, "loss/policy_avg": -0.05416979640722275, "loss/value_avg": 0.5626612901687622, "val/clipfrac_avg": 0.02293882891535759, "policy/entropy_avg": 0.19644975662231445, "val/ratio": 0.9861206412315369, "val/ratio_var": 7.48153543099761e-05, "val/num_eos_tokens": 539, "lr": 1.1927702081543279e-06, "episode": 15600, "epoch": 0.7878787878787878, "step": 975}, {"eps": 0, "objective/kl": 36.37082290649414, "objective/entropy": -29.001220703125, "objective/non_score_reward": -1.8185412883758545, "objective/rlhf_reward": 12.142396926879883, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0011397147318348289, "policy/clipfrac_avg": 0.0031259057577699423, "loss/policy_avg": 0.05607307702302933, "loss/value_avg": 0.48446062207221985, "val/clipfrac_avg": 0.005424283444881439, "policy/entropy_avg": 0.22170257568359375, "val/ratio": 0.9855947494506836, "val/ratio_var": 6.33225790807046e-05, "val/num_eos_tokens": 387, "lr": 1.1841253445542605e-06, "episode": 15616, "epoch": 0.7886868686868687, "step": 976}, {"eps": 0, "objective/kl": 37.47486114501953, "objective/entropy": -60.56960678100586, "objective/non_score_reward": -1.8737430572509766, "objective/rlhf_reward": 11.046178817749023, "objective/scores": 12.9375, "policy/approxkl_avg": 0.0009815995581448078, "policy/clipfrac_avg": 0.005213981028646231, "loss/policy_avg": -0.15400168299674988, "loss/value_avg": 0.8165977001190186, "val/clipfrac_avg": 0.022167012095451355, "policy/entropy_avg": 0.21601486206054688, "val/ratio": 0.9861747026443481, "val/ratio_var": 7.86496966611594e-05, "val/num_eos_tokens": 93, "lr": 1.1755077129169284e-06, "episode": 15632, "epoch": 0.7894949494949495, "step": 977}, {"eps": 0, "objective/kl": 37.225948333740234, "objective/entropy": -56.807586669921875, "objective/non_score_reward": -1.861297607421875, "objective/rlhf_reward": 14.388702392578125, "objective/scores": 16.25, "policy/approxkl_avg": 0.00102808338124305, "policy/clipfrac_avg": 0.0037505230866372585, "loss/policy_avg": -0.1725654900074005, "loss/value_avg": 0.487238347530365, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20328998565673828, "val/ratio": 0.9871898293495178, "val/ratio_var": 9.372083150083199e-05, "val/num_eos_tokens": 128, "lr": 1.166917374742e-06, "episode": 15648, "epoch": 0.7903030303030303, "step": 978}, {"eps": 0, "objective/kl": 34.59331512451172, "objective/entropy": -78.30638122558594, "objective/non_score_reward": -1.729665756225586, "objective/rlhf_reward": 15.489084243774414, "objective/scores": 17.25, "policy/approxkl_avg": 0.0008904373971745372, "policy/clipfrac_avg": 0.00305290799587965, "loss/policy_avg": -0.2764543294906616, "loss/value_avg": 0.33185577392578125, "val/clipfrac_avg": 0.09470640122890472, "policy/entropy_avg": 0.17419052124023438, "val/ratio": 0.9881394505500793, "val/ratio_var": 6.588637916138396e-05, "val/num_eos_tokens": 367, "lr": 1.158354391334362e-06, "episode": 15664, "epoch": 0.7911111111111111, "step": 979}, {"eps": 0, "objective/kl": 40.94211959838867, "objective/entropy": -146.6639862060547, "objective/non_score_reward": -2.0471062660217285, "objective/rlhf_reward": 12.40992546081543, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0008619745494797826, "policy/clipfrac_avg": 0.007621705532073975, "loss/policy_avg": 0.049546245485544205, "loss/value_avg": 0.5992924571037292, "val/clipfrac_avg": 0.09697519987821579, "policy/entropy_avg": 0.15411758422851562, "val/ratio": 0.9889684915542603, "val/ratio_var": 5.157575287739746e-05, "val/num_eos_tokens": 405, "lr": 1.1498188238036862e-06, "episode": 15680, "epoch": 0.7919191919191919, "step": 980}, {"eps": 0, "objective/kl": 35.79730224609375, "objective/entropy": -111.55574798583984, "objective/non_score_reward": -1.7898651361465454, "objective/rlhf_reward": 13.764822006225586, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0009224195964634418, "policy/clipfrac_avg": 0.00732937827706337, "loss/policy_avg": 0.7101428508758545, "loss/value_avg": 1.0345678329467773, "val/clipfrac_avg": 0.0015625000232830644, "policy/entropy_avg": 0.1737055778503418, "val/ratio": 0.9885663390159607, "val/ratio_var": 7.395897409878671e-05, "val/num_eos_tokens": 230, "lr": 1.141310733063991e-06, "episode": 15696, "epoch": 0.7927272727272727, "step": 981}, {"eps": 0, "objective/kl": 41.060279846191406, "objective/entropy": -136.7578125, "objective/non_score_reward": -2.053014039993286, "objective/rlhf_reward": 14.564172744750977, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008562670554965734, "policy/clipfrac_avg": 0.005737176164984703, "loss/policy_avg": -0.1941525638103485, "loss/value_avg": 0.4167664647102356, "val/clipfrac_avg": 0.006640523672103882, "policy/entropy_avg": 0.1706523895263672, "val/ratio": 0.9893268346786499, "val/ratio_var": 6.982601917115971e-05, "val/num_eos_tokens": 935, "lr": 1.132830179833207e-06, "episode": 15712, "epoch": 0.7935353535353535, "step": 982}, {"eps": 0, "objective/kl": 38.72102355957031, "objective/entropy": -109.03129577636719, "objective/non_score_reward": -1.9360511302947998, "objective/rlhf_reward": 14.470198631286621, "objective/scores": 16.375, "policy/approxkl_avg": 0.000771217979490757, "policy/clipfrac_avg": 0.002998888725414872, "loss/policy_avg": -0.1428111046552658, "loss/value_avg": 0.2174670249223709, "val/clipfrac_avg": 0.020510124042630196, "policy/entropy_avg": 0.16396713256835938, "val/ratio": 0.989021897315979, "val/ratio_var": 4.0093269490171224e-05, "val/num_eos_tokens": 25, "lr": 1.1243772246327416e-06, "episode": 15728, "epoch": 0.7943434343434344, "step": 983}, {"eps": 0, "objective/kl": 38.89892578125, "objective/entropy": -6.635685920715332, "objective/non_score_reward": -1.9449464082717896, "objective/rlhf_reward": 14.7113037109375, "objective/scores": 16.625, "policy/approxkl_avg": 0.0009513833792880177, "policy/clipfrac_avg": 0.003157425671815872, "loss/policy_avg": 0.010445185005664825, "loss/value_avg": 0.37888824939727783, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1973400115966797, "val/ratio": 0.9875931739807129, "val/ratio_var": 5.873597910976969e-05, "val/num_eos_tokens": 21, "lr": 1.1159519277870507e-06, "episode": 15744, "epoch": 0.7951515151515152, "step": 984}, {"eps": 0, "objective/kl": 31.575908660888672, "objective/entropy": -149.3587646484375, "objective/non_score_reward": -1.5787954330444336, "objective/rlhf_reward": 15.75714111328125, "objective/scores": 17.375, "policy/approxkl_avg": 0.0007162499241530895, "policy/clipfrac_avg": 0.0078894617035985, "loss/policy_avg": -0.28500208258628845, "loss/value_avg": 0.4213254451751709, "val/clipfrac_avg": 0.023666253313422203, "policy/entropy_avg": 0.14484691619873047, "val/ratio": 0.9908950328826904, "val/ratio_var": 5.517086901818402e-05, "val/num_eos_tokens": 155, "lr": 1.1075543494232049e-06, "episode": 15760, "epoch": 0.795959595959596, "step": 985}, {"eps": 0, "objective/kl": 44.143898010253906, "objective/entropy": -90.30593872070312, "objective/non_score_reward": -2.2071948051452637, "objective/rlhf_reward": 12.292805671691895, "objective/scores": 14.5, "policy/approxkl_avg": 0.000890448980499059, "policy/clipfrac_avg": 0.005722125992178917, "loss/policy_avg": -0.016947008669376373, "loss/value_avg": 0.3959222733974457, "val/clipfrac_avg": 0.020597470924258232, "policy/entropy_avg": 0.1587371826171875, "val/ratio": 0.9887195825576782, "val/ratio_var": 5.094557855045423e-05, "val/num_eos_tokens": 327, "lr": 1.0991845494704673e-06, "episode": 15776, "epoch": 0.7967676767676768, "step": 986}, {"eps": 0, "objective/kl": 42.647743225097656, "objective/entropy": -140.4935760498047, "objective/non_score_reward": -2.132387161254883, "objective/rlhf_reward": 13.516050338745117, "objective/scores": 15.625, "policy/approxkl_avg": 0.0008821362862363458, "policy/clipfrac_avg": 0.005631787236779928, "loss/policy_avg": 0.022888287901878357, "loss/value_avg": 0.36602234840393066, "val/clipfrac_avg": 0.012090040370821953, "policy/entropy_avg": 0.1700587272644043, "val/ratio": 0.9886399507522583, "val/ratio_var": 5.682192204403691e-05, "val/num_eos_tokens": 546, "lr": 1.0908425876598512e-06, "episode": 15792, "epoch": 0.7975757575757576, "step": 987}, {"eps": 0, "objective/kl": 36.984073638916016, "objective/entropy": -53.488460540771484, "objective/non_score_reward": -1.8492038249969482, "objective/rlhf_reward": 14.049233436584473, "objective/scores": 15.875, "policy/approxkl_avg": 0.0009789207251742482, "policy/clipfrac_avg": 0.005884348414838314, "loss/policy_avg": 0.19011469185352325, "loss/value_avg": 0.5114719271659851, "val/clipfrac_avg": 0.013353430666029453, "policy/entropy_avg": 0.20690059661865234, "val/ratio": 0.9872037768363953, "val/ratio_var": 6.569022662006319e-05, "val/num_eos_tokens": 353, "lr": 1.082528523523713e-06, "episode": 15808, "epoch": 0.7983838383838384, "step": 988}, {"eps": 0, "objective/kl": 37.34782409667969, "objective/entropy": -56.68682861328125, "objective/non_score_reward": -1.8673913478851318, "objective/rlhf_reward": 13.050577163696289, "objective/scores": 14.9375, "policy/approxkl_avg": 0.000838876876514405, "policy/clipfrac_avg": 0.0064134905114769936, "loss/policy_avg": 0.1753331869840622, "loss/value_avg": 0.3773725628852844, "val/clipfrac_avg": 0.055415935814380646, "policy/entropy_avg": 0.16259098052978516, "val/ratio": 0.9885395169258118, "val/ratio_var": 8.253273699665442e-05, "val/num_eos_tokens": 32, "lr": 1.0742424163953096e-06, "episode": 15824, "epoch": 0.7991919191919192, "step": 989}, {"eps": 0, "objective/kl": 34.28784942626953, "objective/entropy": -84.58714294433594, "objective/non_score_reward": -1.7143926620483398, "objective/rlhf_reward": 13.20748233795166, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0009669119026511908, "policy/clipfrac_avg": 0.00355747202411294, "loss/policy_avg": -0.013544011861085892, "loss/value_avg": 0.5490726828575134, "val/clipfrac_avg": 0.015838205814361572, "policy/entropy_avg": 0.18613052368164062, "val/ratio": 0.9877301454544067, "val/ratio_var": 5.967232209513895e-05, "val/num_eos_tokens": 115, "lr": 1.0659843254083919e-06, "episode": 15840, "epoch": 0.8, "step": 990}, {"eps": 0, "objective/kl": 35.57737731933594, "objective/entropy": -134.7545928955078, "objective/non_score_reward": -1.7788686752319336, "objective/rlhf_reward": 15.010193824768066, "objective/scores": 16.75, "policy/approxkl_avg": 0.0008520459523424506, "policy/clipfrac_avg": 0.007036273833364248, "loss/policy_avg": 0.17707303166389465, "loss/value_avg": 0.24937717616558075, "val/clipfrac_avg": 0.0005040322430431843, "policy/entropy_avg": 0.14983177185058594, "val/ratio": 0.9898406863212585, "val/ratio_var": 4.8867601435631514e-05, "val/num_eos_tokens": 450, "lr": 1.0577543094967613e-06, "episode": 15856, "epoch": 0.8008080808080809, "step": 991}, {"eps": 0, "objective/kl": 37.30508804321289, "objective/entropy": -99.90908813476562, "objective/non_score_reward": -1.865254521369934, "objective/rlhf_reward": 12.759745597839355, "objective/scores": 14.625, "policy/approxkl_avg": 0.0008709601243026555, "policy/clipfrac_avg": 0.004131123423576355, "loss/policy_avg": -0.1231779083609581, "loss/value_avg": 0.27385565638542175, "val/clipfrac_avg": 0.01071952749043703, "policy/entropy_avg": 0.17008209228515625, "val/ratio": 0.9894505739212036, "val/ratio_var": 6.905445479787886e-05, "val/num_eos_tokens": 376, "lr": 1.0495524273938735e-06, "episode": 15872, "epoch": 0.8016161616161617, "step": 992}, {"eps": 0, "objective/kl": 39.19951629638672, "objective/entropy": -89.06427764892578, "objective/non_score_reward": -1.9599759578704834, "objective/rlhf_reward": 14.532211303710938, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009216031758114696, "policy/clipfrac_avg": 0.00672511849552393, "loss/policy_avg": -0.11878392100334167, "loss/value_avg": 0.6012347340583801, "val/clipfrac_avg": 0.0031250000465661287, "policy/entropy_avg": 0.1765270233154297, "val/ratio": 0.9882692098617554, "val/ratio_var": 3.506496432237327e-05, "val/num_eos_tokens": 74, "lr": 1.041378737632402e-06, "episode": 15888, "epoch": 0.8024242424242424, "step": 993}, {"eps": 0, "objective/kl": 37.94336700439453, "objective/entropy": -48.50525665283203, "objective/non_score_reward": -1.8971682786941528, "objective/rlhf_reward": 15.641894340515137, "objective/scores": 17.5, "policy/approxkl_avg": 0.0010036650346592069, "policy/clipfrac_avg": 0.0061247204430401325, "loss/policy_avg": 0.11932457238435745, "loss/value_avg": 0.28720468282699585, "val/clipfrac_avg": 0.01019021775573492, "policy/entropy_avg": 0.18225669860839844, "val/ratio": 0.9874961972236633, "val/ratio_var": 5.3736544941784814e-05, "val/num_eos_tokens": 8, "lr": 1.0332332985438248e-06, "episode": 15904, "epoch": 0.8032323232323232, "step": 994}, {"eps": 0, "objective/kl": 41.1889762878418, "objective/entropy": -108.57736206054688, "objective/non_score_reward": -2.059448719024658, "objective/rlhf_reward": 14.182738304138184, "objective/scores": 16.25, "policy/approxkl_avg": 0.001018329057842493, "policy/clipfrac_avg": 0.0034168935380876064, "loss/policy_avg": 0.14462614059448242, "loss/value_avg": 0.3247252106666565, "val/clipfrac_avg": 0.00548538425937295, "policy/entropy_avg": 0.20872879028320312, "val/ratio": 0.9865425825119019, "val/ratio_var": 8.348035044036806e-05, "val/num_eos_tokens": 8, "lr": 1.0251161682580125e-06, "episode": 15920, "epoch": 0.804040404040404, "step": 995}, {"eps": 0, "objective/kl": 45.49590301513672, "objective/entropy": -43.08762741088867, "objective/non_score_reward": -2.2747950553894043, "objective/rlhf_reward": 13.912704467773438, "objective/scores": 16.25, "policy/approxkl_avg": 0.0010373364202678204, "policy/clipfrac_avg": 0.006608482450246811, "loss/policy_avg": 0.2817779779434204, "loss/value_avg": 0.49794071912765503, "val/clipfrac_avg": 0.03890983760356903, "policy/entropy_avg": 0.2166609764099121, "val/ratio": 0.986150860786438, "val/ratio_var": 8.553406951250508e-05, "val/num_eos_tokens": 11, "lr": 1.0170274047028068e-06, "episode": 15936, "epoch": 0.8048484848484848, "step": 996}, {"eps": 0, "objective/kl": 35.899574279785156, "objective/entropy": -114.1541976928711, "objective/non_score_reward": -1.7949787378311157, "objective/rlhf_reward": 14.869083404541016, "objective/scores": 16.625, "policy/approxkl_avg": 0.0007367507787421346, "policy/clipfrac_avg": 0.0037586798425763845, "loss/policy_avg": -0.22047999501228333, "loss/value_avg": 0.5745018720626831, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15613555908203125, "val/ratio": 0.9902479648590088, "val/ratio_var": 5.6676559324841946e-05, "val/num_eos_tokens": 560, "lr": 1.0089670656036137e-06, "episode": 15952, "epoch": 0.8056565656565656, "step": 997}, {"eps": 0, "objective/kl": 39.436988830566406, "objective/entropy": -112.42098999023438, "objective/non_score_reward": -1.9718495607376099, "objective/rlhf_reward": 14.48127555847168, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009028672939166427, "policy/clipfrac_avg": 0.0059207589365541935, "loss/policy_avg": -0.04391951486468315, "loss/value_avg": 0.6516869068145752, "val/clipfrac_avg": 0.0127427838742733, "policy/entropy_avg": 0.18039512634277344, "val/ratio": 0.9878066778182983, "val/ratio_var": 8.427277498412877e-05, "val/num_eos_tokens": 515, "lr": 1.0009352084829871e-06, "episode": 15968, "epoch": 0.8064646464646464, "step": 998}, {"eps": 0, "objective/kl": 35.111915588378906, "objective/entropy": -115.74237060546875, "objective/non_score_reward": -1.7555959224700928, "objective/rlhf_reward": 14.103778839111328, "objective/scores": 15.875, "policy/approxkl_avg": 0.0006586643867194653, "policy/clipfrac_avg": 0.006481423042714596, "loss/policy_avg": -0.5726251602172852, "loss/value_avg": 0.6322129964828491, "val/clipfrac_avg": 0.019941631704568863, "policy/entropy_avg": 0.1394972801208496, "val/ratio": 0.9906902313232422, "val/ratio_var": 5.2608244004659355e-05, "val/num_eos_tokens": 81, "lr": 9.929318906602176e-07, "episode": 15984, "epoch": 0.8072727272727273, "step": 999}, {"eps": 0, "objective/kl": 36.119667053222656, "objective/entropy": -35.66822052001953, "objective/non_score_reward": -1.805983304977417, "objective/rlhf_reward": 12.842453956604004, "objective/scores": 14.625, "policy/approxkl_avg": 0.0008497819071635604, "policy/clipfrac_avg": 0.0041199941188097, "loss/policy_avg": -0.1571139395236969, "loss/value_avg": 0.8596901297569275, "val/clipfrac_avg": 0.022714650258421898, "policy/entropy_avg": 0.16556644439697266, "val/ratio": 0.9879409074783325, "val/ratio_var": 6.342474807752296e-05, "val/num_eos_tokens": 82, "lr": 9.849571692509257e-07, "episode": 16000, "epoch": 0.8080808080808081, "step": 1000}, {"eps": 0, "objective/kl": 33.91107177734375, "objective/entropy": -112.11347961425781, "objective/non_score_reward": -1.6955535411834717, "objective/rlhf_reward": 13.304447174072266, "objective/scores": 15.0, "policy/approxkl_avg": 0.0008447067812085152, "policy/clipfrac_avg": 0.004359187558293343, "loss/policy_avg": -0.10610047727823257, "loss/value_avg": 0.5593327283859253, "val/clipfrac_avg": 0.0036525975447148085, "policy/entropy_avg": 0.18085873126983643, "val/ratio": 0.9888993501663208, "val/ratio_var": 0.00012090770906070247, "val/num_eos_tokens": 37, "lr": 9.770111011666582e-07, "episode": 16016, "epoch": 0.8088888888888889, "step": 1001}, {"eps": 0, "objective/kl": 32.61180877685547, "objective/entropy": -143.82159423828125, "objective/non_score_reward": -1.6305904388427734, "objective/rlhf_reward": 13.322534561157227, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0007577177020721138, "policy/clipfrac_avg": 0.005468646064400673, "loss/policy_avg": -0.4011276960372925, "loss/value_avg": 0.8269133567810059, "val/clipfrac_avg": 0.016304347664117813, "policy/entropy_avg": 0.14389371871948242, "val/ratio": 0.990355908870697, "val/ratio_var": 4.9998925533145666e-05, "val/num_eos_tokens": 153, "lr": 9.690937431144725e-07, "episode": 16032, "epoch": 0.8096969696969697, "step": 1002}, {"eps": 0, "objective/kl": 43.84602355957031, "objective/entropy": -41.019737243652344, "objective/non_score_reward": -2.1923012733459473, "objective/rlhf_reward": 14.870198249816895, "objective/scores": 17.0, "policy/approxkl_avg": 0.0009248010464943945, "policy/clipfrac_avg": 0.004084739368408918, "loss/policy_avg": -0.18046468496322632, "loss/value_avg": 0.4406319260597229, "val/clipfrac_avg": 0.01253926008939743, "policy/entropy_avg": 0.1930408477783203, "val/ratio": 0.9877877235412598, "val/ratio_var": 4.6758981625316665e-05, "val/num_eos_tokens": 425, "lr": 9.612051515965388e-07, "episode": 16048, "epoch": 0.8105050505050505, "step": 1003}, {"eps": 0, "objective/kl": 43.825538635253906, "objective/entropy": -115.09342956542969, "objective/non_score_reward": -2.191277027130127, "objective/rlhf_reward": 14.941534996032715, "objective/scores": 17.125, "policy/approxkl_avg": 0.0008623325265944004, "policy/clipfrac_avg": 0.004582014866173267, "loss/policy_avg": -0.03701537847518921, "loss/value_avg": 0.4312940835952759, "val/clipfrac_avg": 0.02018960565328598, "policy/entropy_avg": 0.17541944980621338, "val/ratio": 0.988018274307251, "val/ratio_var": 5.916683949180879e-05, "val/num_eos_tokens": 259, "lr": 9.533453829097355e-07, "episode": 16064, "epoch": 0.8113131313131313, "step": 1004}, {"eps": 0, "objective/kl": 30.437484741210938, "objective/entropy": -62.099971771240234, "objective/non_score_reward": -1.5218743085861206, "objective/rlhf_reward": 13.712501525878906, "objective/scores": 15.25, "policy/approxkl_avg": 0.0007971136947162449, "policy/clipfrac_avg": 0.008183586411178112, "loss/policy_avg": 0.30280348658561707, "loss/value_avg": 0.7898949384689331, "val/clipfrac_avg": 0.003989361692219973, "policy/entropy_avg": 0.1460132598876953, "val/ratio": 0.9896875023841858, "val/ratio_var": 4.330440060584806e-05, "val/num_eos_tokens": 215, "lr": 9.455144931452459e-07, "episode": 16080, "epoch": 0.8121212121212121, "step": 1005}, {"eps": 0, "objective/kl": 33.718963623046875, "objective/entropy": -46.674827575683594, "objective/non_score_reward": -1.6859480142593384, "objective/rlhf_reward": 14.86874008178711, "objective/scores": 16.5, "policy/approxkl_avg": 0.00085799815133214, "policy/clipfrac_avg": 0.003210378112271428, "loss/policy_avg": -0.23599471151828766, "loss/value_avg": 1.1437101364135742, "val/clipfrac_avg": 0.010907451622188091, "policy/entropy_avg": 0.19154703617095947, "val/ratio": 0.9871425628662109, "val/ratio_var": 6.926390778971836e-05, "val/num_eos_tokens": 259, "lr": 9.377125381881607e-07, "episode": 16096, "epoch": 0.812929292929293, "step": 1006}, {"eps": 0, "objective/kl": 35.53752899169922, "objective/entropy": -87.60648345947266, "objective/non_score_reward": -1.7768765687942505, "objective/rlhf_reward": 15.644998550415039, "objective/scores": 17.375, "policy/approxkl_avg": 0.0007149496814236045, "policy/clipfrac_avg": 0.002271715085953474, "loss/policy_avg": -0.02023145742714405, "loss/value_avg": 1.2994359731674194, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1492328643798828, "val/ratio": 0.9887728691101074, "val/ratio_var": 5.388016870710999e-05, "val/num_eos_tokens": 12, "lr": 9.299395737170758e-07, "episode": 16112, "epoch": 0.8137373737373738, "step": 1007}, {"eps": 0, "objective/kl": 36.12407302856445, "objective/entropy": -63.064239501953125, "objective/non_score_reward": -1.8062036037445068, "objective/rlhf_reward": 15.146921157836914, "objective/scores": 17.0, "policy/approxkl_avg": 0.0008970820344984531, "policy/clipfrac_avg": 0.004715266637504101, "loss/policy_avg": 0.022628188133239746, "loss/value_avg": 1.1084355115890503, "val/clipfrac_avg": 0.008854166604578495, "policy/entropy_avg": 0.1754002571105957, "val/ratio": 0.9880603551864624, "val/ratio_var": 8.214469562517479e-05, "val/num_eos_tokens": 540, "lr": 9.221956552036992e-07, "episode": 16128, "epoch": 0.8145454545454546, "step": 1008}, {"eps": 0, "objective/kl": 38.30896759033203, "objective/entropy": -104.08320617675781, "objective/non_score_reward": -1.915448546409607, "objective/rlhf_reward": 13.201739311218262, "objective/scores": 15.125, "policy/approxkl_avg": 0.0008289464749395847, "policy/clipfrac_avg": 0.0036508054472506046, "loss/policy_avg": -0.06129603087902069, "loss/value_avg": 0.558459997177124, "val/clipfrac_avg": 0.013535099104046822, "policy/entropy_avg": 0.16502058506011963, "val/ratio": 0.9892135858535767, "val/ratio_var": 6.506205681944266e-05, "val/num_eos_tokens": 360, "lr": 9.144808379124509e-07, "episode": 16144, "epoch": 0.8153535353535354, "step": 1009}, {"eps": 0, "objective/kl": 42.83964538574219, "objective/entropy": -71.51614379882812, "objective/non_score_reward": -2.141982078552246, "objective/rlhf_reward": 14.701767921447754, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009444506140425801, "policy/clipfrac_avg": 0.002944172825664282, "loss/policy_avg": -0.06057693436741829, "loss/value_avg": 0.288950115442276, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1825122833251953, "val/ratio": 0.9874241352081299, "val/ratio_var": 3.9652099076192826e-05, "val/num_eos_tokens": 613, "lr": 9.067951769000699e-07, "episode": 16160, "epoch": 0.8161616161616162, "step": 1010}, {"eps": 0, "objective/kl": 43.46244812011719, "objective/entropy": -63.48155212402344, "objective/non_score_reward": -2.1731224060058594, "objective/rlhf_reward": 14.65500259399414, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009073936380445957, "policy/clipfrac_avg": 0.005083385389298201, "loss/policy_avg": -0.0031431764364242554, "loss/value_avg": 0.5874146223068237, "val/clipfrac_avg": 0.0003350692568346858, "policy/entropy_avg": 0.1724100112915039, "val/ratio": 0.9878172874450684, "val/ratio_var": 5.2520466852001846e-05, "val/num_eos_tokens": 564, "lr": 8.991387270152202e-07, "episode": 16176, "epoch": 0.816969696969697, "step": 1011}, {"eps": 0, "objective/kl": 30.50366973876953, "objective/entropy": -24.23276138305664, "objective/non_score_reward": -1.5251834392547607, "objective/rlhf_reward": 14.45919132232666, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009571783011779189, "policy/clipfrac_avg": 0.004434539936482906, "loss/policy_avg": 0.020213183015584946, "loss/value_avg": 0.3557816445827484, "val/clipfrac_avg": 0.015398487448692322, "policy/entropy_avg": 0.18828392028808594, "val/ratio": 0.9878268241882324, "val/ratio_var": 6.845602911198512e-05, "val/num_eos_tokens": 104, "lr": 8.915115428981064e-07, "episode": 16192, "epoch": 0.8177777777777778, "step": 1012}, {"eps": 0, "objective/kl": 35.868072509765625, "objective/entropy": -72.25382995605469, "objective/non_score_reward": -1.7934037446975708, "objective/rlhf_reward": 14.183158874511719, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008664189372211695, "policy/clipfrac_avg": 0.0032933237962424755, "loss/policy_avg": -0.06331466138362885, "loss/value_avg": 0.6016228795051575, "val/clipfrac_avg": 0.006926534231752157, "policy/entropy_avg": 0.17766380310058594, "val/ratio": 0.987805962562561, "val/ratio_var": 6.278733781073242e-05, "val/num_eos_tokens": 294, "lr": 8.839136789800729e-07, "episode": 16208, "epoch": 0.8185858585858586, "step": 1013}, {"eps": 0, "objective/kl": 30.213661193847656, "objective/entropy": -88.55311584472656, "objective/non_score_reward": -1.5106831789016724, "objective/rlhf_reward": 13.747129440307617, "objective/scores": 15.25, "policy/approxkl_avg": 0.0007972473395057023, "policy/clipfrac_avg": 0.00806876365095377, "loss/policy_avg": 0.06992897391319275, "loss/value_avg": 0.47664809226989746, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1417522430419922, "val/ratio": 0.9906024932861328, "val/ratio_var": 5.377350316848606e-05, "val/num_eos_tokens": 51, "lr": 8.76345189483222e-07, "episode": 16224, "epoch": 0.8193939393939393, "step": 1014}, {"eps": 0, "objective/kl": 36.9803466796875, "objective/entropy": -92.63037109375, "objective/non_score_reward": -1.84901762008667, "objective/rlhf_reward": 14.065045356750488, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0008528477046638727, "policy/clipfrac_avg": 0.0033233293797820807, "loss/policy_avg": 0.11788127571344376, "loss/value_avg": 0.528445839881897, "val/clipfrac_avg": 0.013305861502885818, "policy/entropy_avg": 0.17418956756591797, "val/ratio": 0.989635169506073, "val/ratio_var": 4.9907346692634746e-05, "val/num_eos_tokens": 387, "lr": 8.688061284200266e-07, "episode": 16240, "epoch": 0.8202020202020202, "step": 1015}, {"eps": 0, "objective/kl": 44.77581024169922, "objective/entropy": -62.09973907470703, "objective/non_score_reward": -2.23879075050354, "objective/rlhf_reward": 12.683084487915039, "objective/scores": 14.9375, "policy/approxkl_avg": 0.001046380726620555, "policy/clipfrac_avg": 0.00542850699275732, "loss/policy_avg": -0.04949585720896721, "loss/value_avg": 0.3462344706058502, "val/clipfrac_avg": 0.03059275820851326, "policy/entropy_avg": 0.2170124053955078, "val/ratio": 0.9858708381652832, "val/ratio_var": 6.539487367263064e-05, "val/num_eos_tokens": 270, "lr": 8.61296549592941e-07, "episode": 16256, "epoch": 0.821010101010101, "step": 1016}, {"eps": 0, "objective/kl": 34.462608337402344, "objective/entropy": -65.8448486328125, "objective/non_score_reward": -1.7231305837631226, "objective/rlhf_reward": 13.925307273864746, "objective/scores": 15.625, "policy/approxkl_avg": 0.000925581669434905, "policy/clipfrac_avg": 0.00542820617556572, "loss/policy_avg": 0.03657815232872963, "loss/value_avg": 0.687595546245575, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19491100311279297, "val/ratio": 0.9877764582633972, "val/ratio_var": 6.733841291861609e-05, "val/num_eos_tokens": 15, "lr": 8.538165065940263e-07, "episode": 16272, "epoch": 0.8218181818181818, "step": 1017}, {"eps": 0, "objective/kl": 41.799583435058594, "objective/entropy": -21.665847778320312, "objective/non_score_reward": -2.0899791717529297, "objective/rlhf_reward": 14.79283332824707, "objective/scores": 16.875, "policy/approxkl_avg": 0.0010365031193941832, "policy/clipfrac_avg": 0.00511179631575942, "loss/policy_avg": -0.097564198076725, "loss/value_avg": 0.5997859835624695, "val/clipfrac_avg": 0.008413461968302727, "policy/entropy_avg": 0.20884323120117188, "val/ratio": 0.9861449003219604, "val/ratio_var": 5.511847120942548e-05, "val/num_eos_tokens": 123, "lr": 8.463660528045514e-07, "episode": 16288, "epoch": 0.8226262626262626, "step": 1018}, {"eps": 0, "objective/kl": 40.70600891113281, "objective/entropy": -106.4912109375, "objective/non_score_reward": -2.0353002548217773, "objective/rlhf_reward": 13.745949745178223, "objective/scores": 15.75, "policy/approxkl_avg": 0.0008316466701216996, "policy/clipfrac_avg": 0.0057511827908456326, "loss/policy_avg": -0.06873838603496552, "loss/value_avg": 0.2803807854652405, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1492767333984375, "val/ratio": 0.9896730184555054, "val/ratio_var": 4.522735616774298e-05, "val/num_eos_tokens": 205, "lr": 8.389452413946314e-07, "episode": 16304, "epoch": 0.8234343434343434, "step": 1019}, {"eps": 0, "objective/kl": 32.72511291503906, "objective/entropy": -29.912065505981445, "objective/non_score_reward": -1.6362557411193848, "objective/rlhf_reward": 13.707494735717773, "objective/scores": 15.375, "policy/approxkl_avg": 0.0009581202175468206, "policy/clipfrac_avg": 0.0044068810530006886, "loss/policy_avg": 0.018638040870428085, "loss/value_avg": 0.7235357761383057, "val/clipfrac_avg": 0.009765625, "policy/entropy_avg": 0.2027587890625, "val/ratio": 0.9886679649353027, "val/ratio_var": 7.175742939580232e-05, "val/num_eos_tokens": 35, "lr": 8.315541253228332e-07, "episode": 16320, "epoch": 0.8242424242424242, "step": 1020}, {"eps": 0, "objective/kl": 44.23792266845703, "objective/entropy": -17.822988510131836, "objective/non_score_reward": -2.2118961811065674, "objective/rlhf_reward": 13.405291557312012, "objective/scores": 15.625, "policy/approxkl_avg": 0.000986535451374948, "policy/clipfrac_avg": 0.0043609049171209335, "loss/policy_avg": 0.15834374725818634, "loss/value_avg": 0.9719556570053101, "val/clipfrac_avg": 0.009959635324776173, "policy/entropy_avg": 0.19144058227539062, "val/ratio": 0.9864338040351868, "val/ratio_var": 2.8990581995458342e-05, "val/num_eos_tokens": 44, "lr": 8.241927573358078e-07, "episode": 16336, "epoch": 0.825050505050505, "step": 1021}, {"eps": 0, "objective/kl": 41.61705017089844, "objective/entropy": -87.69083404541016, "objective/non_score_reward": -2.080852508544922, "objective/rlhf_reward": 14.262897491455078, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008991376380436122, "policy/clipfrac_avg": 0.004512319341301918, "loss/policy_avg": 0.08962170779705048, "loss/value_avg": 0.22370219230651855, "val/clipfrac_avg": 0.04466811940073967, "policy/entropy_avg": 0.190032958984375, "val/ratio": 0.9874501824378967, "val/ratio_var": 6.216822657734156e-05, "val/num_eos_tokens": 17, "lr": 8.168611899679013e-07, "episode": 16352, "epoch": 0.8258585858585858, "step": 1022}, {"eps": 0, "objective/kl": 37.85612487792969, "objective/entropy": -103.04541778564453, "objective/non_score_reward": -1.8928062915802002, "objective/rlhf_reward": 14.255630493164062, "objective/scores": 16.125, "policy/approxkl_avg": 0.0009121837792918086, "policy/clipfrac_avg": 0.004552682861685753, "loss/policy_avg": -0.6490851044654846, "loss/value_avg": 0.6145610213279724, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17924374341964722, "val/ratio": 0.9885929822921753, "val/ratio_var": 8.515298395650461e-05, "val/num_eos_tokens": 624, "lr": 8.095594755407971e-07, "episode": 16368, "epoch": 0.8266666666666667, "step": 1023}, {"eps": 0, "objective/kl": 33.627471923828125, "objective/entropy": -46.773006439208984, "objective/non_score_reward": -1.6813737154006958, "objective/rlhf_reward": 14.521751403808594, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009958728915080428, "policy/clipfrac_avg": 0.0035771888215094805, "loss/policy_avg": 0.09050679206848145, "loss/value_avg": 1.2950150966644287, "val/clipfrac_avg": 0.0034391595982015133, "policy/entropy_avg": 0.17354488372802734, "val/ratio": 0.9874868392944336, "val/ratio_var": 7.542264211224392e-05, "val/num_eos_tokens": 5, "lr": 8.022876661631268e-07, "episode": 16384, "epoch": 0.8274747474747475, "step": 1024}, {"eps": 0, "objective/kl": 40.922630310058594, "objective/entropy": -118.60977172851562, "objective/non_score_reward": -2.0461313724517822, "objective/rlhf_reward": 14.555431365966797, "objective/scores": 16.625, "policy/approxkl_avg": 0.0007377723813988268, "policy/clipfrac_avg": 0.004692882765084505, "loss/policy_avg": 0.09637929499149323, "loss/value_avg": 0.3471735417842865, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1540050506591797, "val/ratio": 0.9898372888565063, "val/ratio_var": 4.050094139529392e-05, "val/num_eos_tokens": 445, "lr": 7.950458137301082e-07, "episode": 16400, "epoch": 0.8282828282828283, "step": 1025}, {"eps": 0, "objective/kl": 47.65478515625, "objective/entropy": 5.97334623336792, "objective/non_score_reward": -2.382739543914795, "objective/rlhf_reward": 15.882884979248047, "objective/scores": 18.25, "policy/approxkl_avg": 0.0011957846581935883, "policy/clipfrac_avg": 0.0038270659279078245, "loss/policy_avg": 0.003938224166631699, "loss/value_avg": 0.12186452746391296, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.24349212646484375, "val/ratio": 0.9836880564689636, "val/ratio_var": 3.132951314910315e-05, "val/num_eos_tokens": 63, "lr": 7.878339699231702e-07, "episode": 16416, "epoch": 0.8290909090909091, "step": 1026}, {"eps": 0, "objective/kl": 43.73109817504883, "objective/entropy": -145.87228393554688, "objective/non_score_reward": -2.1865549087524414, "objective/rlhf_reward": 12.707976341247559, "objective/scores": 14.875, "policy/approxkl_avg": 0.000986664555966854, "policy/clipfrac_avg": 0.005425687879323959, "loss/policy_avg": -0.008172400295734406, "loss/value_avg": 0.40194565057754517, "val/clipfrac_avg": 0.00168609619140625, "policy/entropy_avg": 0.21741676330566406, "val/ratio": 0.9879361987113953, "val/ratio_var": 7.765040209051222e-05, "val/num_eos_tokens": 2002, "lr": 7.806521862095834e-07, "episode": 16432, "epoch": 0.8298989898989899, "step": 1027}, {"eps": 0, "objective/kl": 30.83833122253418, "objective/entropy": -39.62914276123047, "objective/non_score_reward": -1.5419166088104248, "objective/rlhf_reward": 13.919020652770996, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0008982427534647286, "policy/clipfrac_avg": 0.004154416732490063, "loss/policy_avg": -0.040230363607406616, "loss/value_avg": 0.2879355549812317, "val/clipfrac_avg": 0.0003633720916695893, "policy/entropy_avg": 0.1513214111328125, "val/ratio": 0.9900394678115845, "val/ratio_var": 4.5976830733707175e-05, "val/num_eos_tokens": 42, "lr": 7.735005138421003e-07, "episode": 16448, "epoch": 0.8307070707070707, "step": 1028}, {"eps": 0, "objective/kl": 39.669403076171875, "objective/entropy": -91.81086730957031, "objective/non_score_reward": -1.9834702014923096, "objective/rlhf_reward": 14.73528003692627, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009472984238527715, "policy/clipfrac_avg": 0.0033632363192737103, "loss/policy_avg": -0.17712682485580444, "loss/value_avg": 0.41484764218330383, "val/clipfrac_avg": 0.015109826810657978, "policy/entropy_avg": 0.1894826889038086, "val/ratio": 0.9873372316360474, "val/ratio_var": 6.729063170496374e-05, "val/num_eos_tokens": 637, "lr": 7.663790038585794e-07, "episode": 16464, "epoch": 0.8315151515151515, "step": 1029}, {"eps": 0, "objective/kl": 40.63905334472656, "objective/entropy": -81.92291259765625, "objective/non_score_reward": -2.0319526195526123, "objective/rlhf_reward": 14.960234642028809, "objective/scores": 17.0, "policy/approxkl_avg": 0.0009109035599976778, "policy/clipfrac_avg": 0.003938743844628334, "loss/policy_avg": 0.03287976607680321, "loss/value_avg": 0.3657122254371643, "val/clipfrac_avg": 0.005359000526368618, "policy/entropy_avg": 0.17749857902526855, "val/ratio": 0.9891733527183533, "val/ratio_var": 5.9037287428509444e-05, "val/num_eos_tokens": 56, "lr": 7.592877070816268e-07, "episode": 16480, "epoch": 0.8323232323232324, "step": 1030}, {"eps": 0, "objective/kl": 43.41844177246094, "objective/entropy": -79.7171630859375, "objective/non_score_reward": -2.170922040939331, "objective/rlhf_reward": 13.88376522064209, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009897679556161165, "policy/clipfrac_avg": 0.00460906233638525, "loss/policy_avg": 0.054874081164598465, "loss/value_avg": 0.28462013602256775, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18915176391601562, "val/ratio": 0.987737238407135, "val/ratio_var": 6.536820001201704e-05, "val/num_eos_tokens": 413, "lr": 7.522266741182305e-07, "episode": 16496, "epoch": 0.8331313131313132, "step": 1031}, {"eps": 0, "objective/kl": 32.546966552734375, "objective/entropy": -75.63118743896484, "objective/non_score_reward": -1.6273484230041504, "objective/rlhf_reward": 12.075777053833008, "objective/scores": 13.6875, "policy/approxkl_avg": 0.000955111114308238, "policy/clipfrac_avg": 0.005775670520961285, "loss/policy_avg": 0.42096102237701416, "loss/value_avg": 0.7095996141433716, "val/clipfrac_avg": 0.017921777442097664, "policy/entropy_avg": 0.16732215881347656, "val/ratio": 0.989020586013794, "val/ratio_var": 4.018365507363342e-05, "val/num_eos_tokens": 145, "lr": 7.451959553594051e-07, "episode": 16512, "epoch": 0.833939393939394, "step": 1032}, {"eps": 0, "objective/kl": 38.599403381347656, "objective/entropy": -53.61748123168945, "objective/non_score_reward": -1.929970145225525, "objective/rlhf_reward": 12.640341758728027, "objective/scores": 14.5625, "policy/approxkl_avg": 0.0008390340954065323, "policy/clipfrac_avg": 0.0041229561902582645, "loss/policy_avg": 0.017501141875982285, "loss/value_avg": 0.4045110046863556, "val/clipfrac_avg": 0.020542385056614876, "policy/entropy_avg": 0.18261051177978516, "val/ratio": 0.988516092300415, "val/ratio_var": 9.491335367783904e-05, "val/num_eos_tokens": 139, "lr": 7.381956009798253e-07, "episode": 16528, "epoch": 0.8347474747474748, "step": 1033}, {"eps": 0, "objective/kl": 36.218544006347656, "objective/entropy": -103.81524658203125, "objective/non_score_reward": -1.810927152633667, "objective/rlhf_reward": 14.07969856262207, "objective/scores": 15.875, "policy/approxkl_avg": 0.0007683830335736275, "policy/clipfrac_avg": 0.003070900682359934, "loss/policy_avg": -0.03480757772922516, "loss/value_avg": 1.401247501373291, "val/clipfrac_avg": 0.03183094039559364, "policy/entropy_avg": 0.14527511596679688, "val/ratio": 0.9899626970291138, "val/ratio_var": 6.579139153473079e-05, "val/num_eos_tokens": 65, "lr": 7.312256609374708e-07, "episode": 16544, "epoch": 0.8355555555555556, "step": 1034}, {"eps": 0, "objective/kl": 38.71485900878906, "objective/entropy": -59.279266357421875, "objective/non_score_reward": -1.935742974281311, "objective/rlhf_reward": 14.07206916809082, "objective/scores": 16.0, "policy/approxkl_avg": 0.0007580246892757714, "policy/clipfrac_avg": 0.003035132307559252, "loss/policy_avg": -0.28031888604164124, "loss/value_avg": 1.1235523223876953, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14683592319488525, "val/ratio": 0.9899235963821411, "val/ratio_var": 5.906899241381325e-05, "val/num_eos_tokens": 109, "lr": 7.242861849732696e-07, "episode": 16560, "epoch": 0.8363636363636363, "step": 1035}, {"eps": 0, "objective/kl": 38.85520553588867, "objective/entropy": -64.92012023925781, "objective/non_score_reward": -1.9427603483200073, "objective/rlhf_reward": 14.400989532470703, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009576823795214295, "policy/clipfrac_avg": 0.004880891181528568, "loss/policy_avg": -0.14042839407920837, "loss/value_avg": 1.1183263063430786, "val/clipfrac_avg": 0.012853971682488918, "policy/entropy_avg": 0.19240808486938477, "val/ratio": 0.987326979637146, "val/ratio_var": 9.263003448722884e-05, "val/num_eos_tokens": 155, "lr": 7.173772226107434e-07, "episode": 16576, "epoch": 0.8371717171717171, "step": 1036}, {"eps": 0, "objective/kl": 36.47294616699219, "objective/entropy": -48.31360626220703, "objective/non_score_reward": -1.8236472606658936, "objective/rlhf_reward": 14.137290000915527, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0008876229985617101, "policy/clipfrac_avg": 0.002901179250329733, "loss/policy_avg": -0.03311082720756531, "loss/value_avg": 0.3677228093147278, "val/clipfrac_avg": 0.002261103130877018, "policy/entropy_avg": 0.20080184936523438, "val/ratio": 0.9893977642059326, "val/ratio_var": 5.7600576838012785e-05, "val/num_eos_tokens": 20, "lr": 7.104988231556543e-07, "episode": 16592, "epoch": 0.8379797979797979, "step": 1037}, {"eps": 0, "objective/kl": 41.474632263183594, "objective/entropy": -66.36492919921875, "objective/non_score_reward": -2.0737314224243164, "objective/rlhf_reward": 15.168456077575684, "objective/scores": 17.25, "policy/approxkl_avg": 0.0009756431682035327, "policy/clipfrac_avg": 0.005543471779674292, "loss/policy_avg": -0.10436652600765228, "loss/value_avg": 0.43171289563179016, "val/clipfrac_avg": 0.01755008101463318, "policy/entropy_avg": 0.188018798828125, "val/ratio": 0.9874497652053833, "val/ratio_var": 5.146233888808638e-05, "val/num_eos_tokens": 3, "lr": 7.036510356956494e-07, "episode": 16608, "epoch": 0.8387878787878787, "step": 1038}, {"eps": 0, "objective/kl": 33.46930694580078, "objective/entropy": -82.42080688476562, "objective/non_score_reward": -1.6734652519226074, "objective/rlhf_reward": 16.849971771240234, "objective/scores": 18.5, "policy/approxkl_avg": 0.0009166139643639326, "policy/clipfrac_avg": 0.0028331982903182507, "loss/policy_avg": -0.23364350199699402, "loss/value_avg": 0.3087168335914612, "val/clipfrac_avg": 0.022823384031653404, "policy/entropy_avg": 0.18764019012451172, "val/ratio": 0.9876014590263367, "val/ratio_var": 7.868735701777041e-05, "val/num_eos_tokens": 150, "lr": 6.968339090999188e-07, "episode": 16624, "epoch": 0.8395959595959596, "step": 1039}, {"eps": 0, "objective/kl": 31.56005096435547, "objective/entropy": -98.47803497314453, "objective/non_score_reward": -1.5780024528503418, "objective/rlhf_reward": 12.9376220703125, "objective/scores": 14.5, "policy/approxkl_avg": 0.0007581596728414297, "policy/clipfrac_avg": 0.0029252313543111086, "loss/policy_avg": -0.24348971247673035, "loss/value_avg": 0.46729177236557007, "val/clipfrac_avg": 0.018430940806865692, "policy/entropy_avg": 0.14857721328735352, "val/ratio": 0.9900062084197998, "val/ratio_var": 7.637133239768445e-05, "val/num_eos_tokens": 64, "lr": 6.900474920188383e-07, "episode": 16640, "epoch": 0.8404040404040404, "step": 1040}, {"eps": 0, "objective/kl": 32.892269134521484, "objective/entropy": -67.75382995605469, "objective/non_score_reward": -1.64461350440979, "objective/rlhf_reward": 14.644449234008789, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009258019272238016, "policy/clipfrac_avg": 0.004761096090078354, "loss/policy_avg": -0.3006308078765869, "loss/value_avg": 0.9436385035514832, "val/clipfrac_avg": 0.018809743225574493, "policy/entropy_avg": 0.1688389778137207, "val/ratio": 0.9893865585327148, "val/ratio_var": 6.368513277266175e-05, "val/num_eos_tokens": 357, "lr": 6.832918328836247e-07, "episode": 16656, "epoch": 0.8412121212121212, "step": 1041}, {"eps": 0, "objective/kl": 35.5787467956543, "objective/entropy": -74.0018310546875, "objective/non_score_reward": -1.7789373397827148, "objective/rlhf_reward": 13.291374206542969, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0008827393758110702, "policy/clipfrac_avg": 0.004510640166699886, "loss/policy_avg": -0.06437136977910995, "loss/value_avg": 0.4288250207901001, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.174835205078125, "val/ratio": 0.9889439344406128, "val/ratio_var": 5.196507117943838e-05, "val/num_eos_tokens": 88, "lr": 6.765669799059915e-07, "episode": 16672, "epoch": 0.842020202020202, "step": 1042}, {"eps": 0, "objective/kl": 36.0411491394043, "objective/entropy": -94.1718521118164, "objective/non_score_reward": -1.8020575046539307, "objective/rlhf_reward": 14.455755233764648, "objective/scores": 16.25, "policy/approxkl_avg": 0.0007288270280696452, "policy/clipfrac_avg": 0.003802281804382801, "loss/policy_avg": 0.08304592221975327, "loss/value_avg": 0.627299427986145, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14253592491149902, "val/ratio": 0.9900709390640259, "val/ratio_var": 3.72955619241111e-05, "val/num_eos_tokens": 492, "lr": 6.698729810778065e-07, "episode": 16688, "epoch": 0.8428282828282828, "step": 1043}, {"eps": 0, "objective/kl": 37.80200958251953, "objective/entropy": -44.56480407714844, "objective/non_score_reward": -1.8901004791259766, "objective/rlhf_reward": 13.203649520874023, "objective/scores": 15.125, "policy/approxkl_avg": 0.0012037131236866117, "policy/clipfrac_avg": 0.005290274508297443, "loss/policy_avg": 0.289115309715271, "loss/value_avg": 0.6000362038612366, "val/clipfrac_avg": 0.0035122863482683897, "policy/entropy_avg": 0.21977996826171875, "val/ratio": 0.9847176671028137, "val/ratio_var": 9.624927770346403e-05, "val/num_eos_tokens": 378, "lr": 6.632098841707458e-07, "episode": 16704, "epoch": 0.8436363636363636, "step": 1044}, {"eps": 0, "objective/kl": 39.69865798950195, "objective/entropy": -68.83721923828125, "objective/non_score_reward": -1.9849328994750977, "objective/rlhf_reward": 13.468192100524902, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0008786184480413795, "policy/clipfrac_avg": 0.004983194172382355, "loss/policy_avg": 0.07545819133520126, "loss/value_avg": 0.46948039531707764, "val/clipfrac_avg": 0.00048828125, "policy/entropy_avg": 0.17647552490234375, "val/ratio": 0.9880126714706421, "val/ratio_var": 4.486176840146072e-05, "val/num_eos_tokens": 99, "lr": 6.565777367359516e-07, "episode": 16720, "epoch": 0.8444444444444444, "step": 1045}, {"eps": 0, "objective/kl": 39.10038375854492, "objective/entropy": -133.97775268554688, "objective/non_score_reward": -1.9550191164016724, "objective/rlhf_reward": 13.568418502807617, "objective/scores": 15.5, "policy/approxkl_avg": 0.0007223144639283419, "policy/clipfrac_avg": 0.0023505911231040955, "loss/policy_avg": 0.18258720636367798, "loss/value_avg": 0.33022794127464294, "val/clipfrac_avg": 0.001006610575132072, "policy/entropy_avg": 0.1742839813232422, "val/ratio": 0.9903188943862915, "val/ratio_var": 4.2478550312807783e-05, "val/num_eos_tokens": 767, "lr": 6.499765861036994e-07, "episode": 16736, "epoch": 0.8452525252525253, "step": 1046}, {"eps": 0, "objective/kl": 40.72428894042969, "objective/entropy": -11.881628036499023, "objective/non_score_reward": -2.036214590072632, "objective/rlhf_reward": 15.112222671508789, "objective/scores": 17.125, "policy/approxkl_avg": 0.0010564490221440792, "policy/clipfrac_avg": 0.007933112792670727, "loss/policy_avg": 0.004041165113449097, "loss/value_avg": 0.2533167600631714, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20357513427734375, "val/ratio": 0.9868043661117554, "val/ratio_var": 6.374649819917977e-05, "val/num_eos_tokens": 42, "lr": 6.43406479383053e-07, "episode": 16752, "epoch": 0.8460606060606061, "step": 1047}, {"eps": 0, "objective/kl": 34.73613739013672, "objective/entropy": -45.377681732177734, "objective/non_score_reward": -1.736806869506836, "objective/rlhf_reward": 14.192880630493164, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0009812908247113228, "policy/clipfrac_avg": 0.0049714683555066586, "loss/policy_avg": 0.0763624757528305, "loss/value_avg": 0.4388294219970703, "val/clipfrac_avg": 0.0006127451197244227, "policy/entropy_avg": 0.18614768981933594, "val/ratio": 0.98753422498703, "val/ratio_var": 4.261466165189631e-05, "val/num_eos_tokens": 351, "lr": 6.368674634615357e-07, "episode": 16768, "epoch": 0.8468686868686869, "step": 1048}, {"eps": 0, "objective/kl": 42.18169021606445, "objective/entropy": -80.03951263427734, "objective/non_score_reward": -2.1090846061706543, "objective/rlhf_reward": 13.812790870666504, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0009452086524106562, "policy/clipfrac_avg": 0.004814188927412033, "loss/policy_avg": -0.12992079555988312, "loss/value_avg": 0.939970850944519, "val/clipfrac_avg": 0.0003124999930150807, "policy/entropy_avg": 0.1846628189086914, "val/ratio": 0.988731324672699, "val/ratio_var": 6.252430466702208e-05, "val/num_eos_tokens": 193, "lr": 6.303595850047844e-07, "episode": 16784, "epoch": 0.8476767676767677, "step": 1049}, {"eps": 0, "objective/kl": 37.25836944580078, "objective/entropy": -56.887001037597656, "objective/non_score_reward": -1.8629186153411865, "objective/rlhf_reward": 14.301143646240234, "objective/scores": 16.125, "policy/approxkl_avg": 0.0008069226169027388, "policy/clipfrac_avg": 0.0021998940501362085, "loss/policy_avg": -0.12597322463989258, "loss/value_avg": 0.5964376926422119, "val/clipfrac_avg": 0.00014467592700384557, "policy/entropy_avg": 0.17313003540039062, "val/ratio": 0.9889218807220459, "val/ratio_var": 4.452227949514054e-05, "val/num_eos_tokens": 69, "lr": 6.238828904562316e-07, "episode": 16800, "epoch": 0.8484848484848485, "step": 1050}, {"eps": 0, "objective/kl": 36.81440734863281, "objective/entropy": -162.38363647460938, "objective/non_score_reward": -1.8407204151153564, "objective/rlhf_reward": 13.549904823303223, "objective/scores": 15.375, "policy/approxkl_avg": 0.0006882078596390784, "policy/clipfrac_avg": 0.0029306234791874886, "loss/policy_avg": -0.284543514251709, "loss/value_avg": 0.4945632815361023, "val/clipfrac_avg": 0.02376413717865944, "policy/entropy_avg": 0.13589084148406982, "val/ratio": 0.9912172555923462, "val/ratio_var": 4.5012933696853e-05, "val/num_eos_tokens": 810, "lr": 6.174374260367611e-07, "episode": 16816, "epoch": 0.8492929292929293, "step": 1051}, {"eps": 0, "objective/kl": 35.95270538330078, "objective/entropy": -106.74056243896484, "objective/non_score_reward": -1.7976351976394653, "objective/rlhf_reward": 16.46017837524414, "objective/scores": 18.25, "policy/approxkl_avg": 0.0009089771774597466, "policy/clipfrac_avg": 0.005288749001920223, "loss/policy_avg": -0.011637904681265354, "loss/value_avg": 0.556053638458252, "val/clipfrac_avg": 0.0026041667442768812, "policy/entropy_avg": 0.18283462524414062, "val/ratio": 0.9882605075836182, "val/ratio_var": 7.140174420783296e-05, "val/num_eos_tokens": 104, "lr": 6.110232377443864e-07, "episode": 16832, "epoch": 0.8501010101010101, "step": 1052}, {"eps": 0, "objective/kl": 40.826602935791016, "objective/entropy": -75.6595458984375, "objective/non_score_reward": -2.041330337524414, "objective/rlhf_reward": 13.857107162475586, "objective/scores": 15.875, "policy/approxkl_avg": 0.0009147390373982489, "policy/clipfrac_avg": 0.0046502938494086266, "loss/policy_avg": -0.09856721758842468, "loss/value_avg": 0.43345025181770325, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21721649169921875, "val/ratio": 0.9873005151748657, "val/ratio_var": 7.706157339271158e-05, "val/num_eos_tokens": 61, "lr": 6.04640371353914e-07, "episode": 16848, "epoch": 0.850909090909091, "step": 1053}, {"eps": 0, "objective/kl": 38.763389587402344, "objective/entropy": -99.58453369140625, "objective/non_score_reward": -1.938169240951538, "objective/rlhf_reward": 15.054018020629883, "objective/scores": 17.0, "policy/approxkl_avg": 0.0008463884005323052, "policy/clipfrac_avg": 0.0038104294799268246, "loss/policy_avg": 0.051140159368515015, "loss/value_avg": 0.46211516857147217, "val/clipfrac_avg": 0.0019132653251290321, "policy/entropy_avg": 0.16744399070739746, "val/ratio": 0.989348292350769, "val/ratio_var": 8.285223157145083e-05, "val/num_eos_tokens": 520, "lr": 5.98288872416628e-07, "episode": 16864, "epoch": 0.8517171717171718, "step": 1054}, {"eps": 0, "objective/kl": 37.5797004699707, "objective/entropy": -79.0547103881836, "objective/non_score_reward": -1.878985047340393, "objective/rlhf_reward": 14.566328048706055, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009661274962127209, "policy/clipfrac_avg": 0.00524417357519269, "loss/policy_avg": -0.03345664590597153, "loss/value_avg": 0.41552969813346863, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.186629980802536, "val/ratio": 0.9881995916366577, "val/ratio_var": 7.692445797147229e-05, "val/num_eos_tokens": 305, "lr": 5.919687862599549e-07, "episode": 16880, "epoch": 0.8525252525252526, "step": 1055}, {"eps": 0, "objective/kl": 38.447975158691406, "objective/entropy": -91.50765991210938, "objective/non_score_reward": -1.9223988056182861, "objective/rlhf_reward": 12.522913932800293, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0007886831881478429, "policy/clipfrac_avg": 0.00426090694963932, "loss/policy_avg": 0.1846567541360855, "loss/value_avg": 0.5820722579956055, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15495681762695312, "val/ratio": 0.9909533262252808, "val/ratio_var": 5.192918251850642e-05, "val/num_eos_tokens": 137, "lr": 5.856801579871457e-07, "episode": 16896, "epoch": 0.8533333333333334, "step": 1056}, {"eps": 0, "objective/kl": 41.44641876220703, "objective/entropy": -45.45872497558594, "objective/non_score_reward": -2.0723209381103516, "objective/rlhf_reward": 13.966741561889648, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009499872685410082, "policy/clipfrac_avg": 0.007878290489315987, "loss/policy_avg": -0.014410331845283508, "loss/value_avg": 0.45579588413238525, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18343734741210938, "val/ratio": 0.9887840747833252, "val/ratio_var": 6.99822703609243e-05, "val/num_eos_tokens": 399, "lr": 5.794230324769518e-07, "episode": 16912, "epoch": 0.8541414141414141, "step": 1057}, {"eps": 0, "objective/kl": 38.324073791503906, "objective/entropy": -55.36639404296875, "objective/non_score_reward": -1.916203498840332, "objective/rlhf_reward": 12.029109001159668, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0009155046427622437, "policy/clipfrac_avg": 0.003906010650098324, "loss/policy_avg": -0.05935902148485184, "loss/value_avg": 0.36739227175712585, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17214488983154297, "val/ratio": 0.9891675114631653, "val/ratio_var": 5.269415851216763e-05, "val/num_eos_tokens": 62, "lr": 5.731974543833041e-07, "episode": 16928, "epoch": 0.8549494949494949, "step": 1058}, {"eps": 0, "objective/kl": 39.27944564819336, "objective/entropy": -77.39466857910156, "objective/non_score_reward": -1.9639724493026733, "objective/rlhf_reward": 12.512590408325195, "objective/scores": 14.5, "policy/approxkl_avg": 0.0011488801101222634, "policy/clipfrac_avg": 0.005929230712354183, "loss/policy_avg": 0.12509797513484955, "loss/value_avg": 0.5901766419410706, "val/clipfrac_avg": 0.01009130198508501, "policy/entropy_avg": 0.22491836547851562, "val/ratio": 0.9847910404205322, "val/ratio_var": 8.84500186657533e-05, "val/num_eos_tokens": 274, "lr": 5.670034681349995e-07, "episode": 16944, "epoch": 0.8557575757575757, "step": 1059}, {"eps": 0, "objective/kl": 34.862098693847656, "objective/entropy": -68.70003509521484, "objective/non_score_reward": -1.7431049346923828, "objective/rlhf_reward": 15.538145065307617, "objective/scores": 17.25, "policy/approxkl_avg": 0.0010132534662261605, "policy/clipfrac_avg": 0.00747870746999979, "loss/policy_avg": 0.027433840557932854, "loss/value_avg": 0.2936461567878723, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19480419158935547, "val/ratio": 0.9880098700523376, "val/ratio_var": 6.574365397682413e-05, "val/num_eos_tokens": 6, "lr": 5.608411179353767e-07, "episode": 16960, "epoch": 0.8565656565656565, "step": 1060}, {"eps": 0, "objective/kl": 36.893959045410156, "objective/entropy": -101.6231918334961, "objective/non_score_reward": -1.844698190689087, "objective/rlhf_reward": 13.850614547729492, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008218783186748624, "policy/clipfrac_avg": 0.005969468504190445, "loss/policy_avg": 0.22168952226638794, "loss/value_avg": 0.6544172763824463, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15513205528259277, "val/ratio": 0.9892902374267578, "val/ratio_var": 6.282331742113456e-05, "val/num_eos_tokens": 465, "lr": 5.547104477620041e-07, "episode": 16976, "epoch": 0.8573737373737373, "step": 1061}, {"eps": 0, "objective/kl": 33.110145568847656, "objective/entropy": -112.72584533691406, "objective/non_score_reward": -1.6555070877075195, "objective/rlhf_reward": 15.28980541229248, "objective/scores": 17.0, "policy/approxkl_avg": 0.0006752893095836043, "policy/clipfrac_avg": 0.0032210415229201317, "loss/policy_avg": -0.23859268426895142, "loss/value_avg": 0.5853025913238525, "val/clipfrac_avg": 4.234417428961024e-05, "policy/entropy_avg": 0.12448978424072266, "val/ratio": 0.9915745258331299, "val/ratio_var": 5.202127795200795e-05, "val/num_eos_tokens": 451, "lr": 5.486115013663668e-07, "episode": 16992, "epoch": 0.8581818181818182, "step": 1062}, {"eps": 0, "objective/kl": 38.634700775146484, "objective/entropy": -49.841190338134766, "objective/non_score_reward": -1.9317350387573242, "objective/rlhf_reward": 16.25576400756836, "objective/scores": 18.25, "policy/approxkl_avg": 0.0011730990372598171, "policy/clipfrac_avg": 0.008051658049225807, "loss/policy_avg": -0.1410359889268875, "loss/value_avg": 0.22414374351501465, "val/clipfrac_avg": 0.009043844416737556, "policy/entropy_avg": 0.22179222106933594, "val/ratio": 0.9854654669761658, "val/ratio_var": 7.81394264777191e-05, "val/num_eos_tokens": 125, "lr": 5.425443222735527e-07, "episode": 17008, "epoch": 0.858989898989899, "step": 1063}, {"eps": 0, "objective/kl": 37.95829772949219, "objective/entropy": -68.24198913574219, "objective/non_score_reward": -1.8979148864746094, "objective/rlhf_reward": 15.38333511352539, "objective/scores": 17.25, "policy/approxkl_avg": 0.0008637227001599967, "policy/clipfrac_avg": 0.006955782417207956, "loss/policy_avg": -0.060928136110305786, "loss/value_avg": 0.45962244272232056, "val/clipfrac_avg": 0.027727147564291954, "policy/entropy_avg": 0.1758570671081543, "val/ratio": 0.9886413812637329, "val/ratio_var": 8.117071411106735e-05, "val/num_eos_tokens": 423, "lr": 5.365089537819435e-07, "episode": 17024, "epoch": 0.8597979797979798, "step": 1064}, {"eps": 0, "objective/kl": 41.044742584228516, "objective/entropy": -66.94715881347656, "objective/non_score_reward": -2.052237033843994, "objective/rlhf_reward": 13.018075942993164, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0009623835212551057, "policy/clipfrac_avg": 0.0037963923532515764, "loss/policy_avg": -0.024640943855047226, "loss/value_avg": 0.736940324306488, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1932220458984375, "val/ratio": 0.9877476692199707, "val/ratio_var": 5.7415742048760876e-05, "val/num_eos_tokens": 86, "lr": 5.305054389629022e-07, "episode": 17040, "epoch": 0.8606060606060606, "step": 1065}, {"eps": 0, "objective/kl": 40.31273651123047, "objective/entropy": -115.46347045898438, "objective/non_score_reward": -2.015636682510376, "objective/rlhf_reward": 13.367176055908203, "objective/scores": 15.375, "policy/approxkl_avg": 0.0008412263705395162, "policy/clipfrac_avg": 0.01834527961909771, "loss/policy_avg": 0.11599922180175781, "loss/value_avg": 1.5807065963745117, "val/clipfrac_avg": 1.0643732821336016e-05, "policy/entropy_avg": 0.17316818237304688, "val/ratio": 0.9887286424636841, "val/ratio_var": 4.431970592122525e-05, "val/num_eos_tokens": 71, "lr": 5.245338206604733e-07, "episode": 17056, "epoch": 0.8614141414141414, "step": 1066}, {"eps": 0, "objective/kl": 39.36028289794922, "objective/entropy": -67.14189147949219, "objective/non_score_reward": -1.9680140018463135, "objective/rlhf_reward": 15.414798736572266, "objective/scores": 17.375, "policy/approxkl_avg": 0.0008963524596765637, "policy/clipfrac_avg": 0.00728583661839366, "loss/policy_avg": -0.11949857324361801, "loss/value_avg": 0.5354464650154114, "val/clipfrac_avg": 0.0072115384973585606, "policy/entropy_avg": 0.17730861902236938, "val/ratio": 0.9881836175918579, "val/ratio_var": 3.5629829653771594e-05, "val/num_eos_tokens": 286, "lr": 5.185941414910673e-07, "episode": 17072, "epoch": 0.8622222222222222, "step": 1067}, {"eps": 0, "objective/kl": 37.194644927978516, "objective/entropy": -115.96305847167969, "objective/non_score_reward": -1.8597323894500732, "objective/rlhf_reward": 12.882455825805664, "objective/scores": 14.75, "policy/approxkl_avg": 0.000633776537142694, "policy/clipfrac_avg": 0.004217084962874651, "loss/policy_avg": 0.10199490189552307, "loss/value_avg": 0.35868531465530396, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.13162803649902344, "val/ratio": 0.9914271235466003, "val/ratio_var": 5.447517105494626e-05, "val/num_eos_tokens": 156, "lr": 5.126864438431628e-07, "episode": 17088, "epoch": 0.863030303030303, "step": 1068}, {"eps": 0, "objective/kl": 30.918790817260742, "objective/entropy": -27.455425262451172, "objective/non_score_reward": -1.545939564704895, "objective/rlhf_reward": 13.969684600830078, "objective/scores": 15.5, "policy/approxkl_avg": 0.000932033930439502, "policy/clipfrac_avg": 0.0023164437152445316, "loss/policy_avg": 0.09657607972621918, "loss/value_avg": 0.33049535751342773, "val/clipfrac_avg": 0.0017710273386910558, "policy/entropy_avg": 0.2050342559814453, "val/ratio": 0.9869778752326965, "val/ratio_var": 7.318282587220892e-05, "val/num_eos_tokens": 117, "lr": 5.06810769877002e-07, "episode": 17104, "epoch": 0.8638383838383838, "step": 1069}, {"eps": 0, "objective/kl": 42.76515197753906, "objective/entropy": -41.616695404052734, "objective/non_score_reward": -2.1382575035095215, "objective/rlhf_reward": 13.17424201965332, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0009901263983920217, "policy/clipfrac_avg": 0.003959751222282648, "loss/policy_avg": -0.07673387229442596, "loss/value_avg": 0.6429687738418579, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1980295181274414, "val/ratio": 0.987491250038147, "val/ratio_var": 4.666786844609305e-05, "val/num_eos_tokens": 199, "lr": 5.009671615242928e-07, "episode": 17120, "epoch": 0.8646464646464647, "step": 1070}, {"eps": 0, "objective/kl": 39.59283447265625, "objective/entropy": -53.619712829589844, "objective/non_score_reward": -1.9796417951583862, "objective/rlhf_reward": 13.887545585632324, "objective/scores": 15.875, "policy/approxkl_avg": 0.0010703656589612365, "policy/clipfrac_avg": 0.0033148752991110086, "loss/policy_avg": -0.03734193369746208, "loss/value_avg": 0.7440835237503052, "val/clipfrac_avg": 0.00032552084303461015, "policy/entropy_avg": 0.19559097290039062, "val/ratio": 0.9871270060539246, "val/ratio_var": 6.861035944893956e-05, "val/num_eos_tokens": 134, "lr": 4.951556604879049e-07, "episode": 17136, "epoch": 0.8654545454545455, "step": 1071}, {"eps": 0, "objective/kl": 33.41657638549805, "objective/entropy": -32.19466781616211, "objective/non_score_reward": -1.6708288192749023, "objective/rlhf_reward": 12.516671180725098, "objective/scores": 14.1875, "policy/approxkl_avg": 0.0010530936997383833, "policy/clipfrac_avg": 0.005051491782069206, "loss/policy_avg": 0.05410761386156082, "loss/value_avg": 0.6214333772659302, "val/clipfrac_avg": 0.006988893263041973, "policy/entropy_avg": 0.18621063232421875, "val/ratio": 0.9861816167831421, "val/ratio_var": 5.813246752950363e-05, "val/num_eos_tokens": 41, "lr": 4.893763082415736e-07, "episode": 17152, "epoch": 0.8662626262626263, "step": 1072}, {"eps": 0, "objective/kl": 44.275638580322266, "objective/entropy": -73.75119018554688, "objective/non_score_reward": -2.2137818336486816, "objective/rlhf_reward": 13.872156143188477, "objective/scores": 16.125, "policy/approxkl_avg": 0.0009736482752487063, "policy/clipfrac_avg": 0.002118697389960289, "loss/policy_avg": -0.13285210728645325, "loss/value_avg": 0.37682080268859863, "val/clipfrac_avg": 0.003361742477864027, "policy/entropy_avg": 0.1989288330078125, "val/ratio": 0.9864859580993652, "val/ratio_var": 7.720785652054474e-05, "val/num_eos_tokens": 178, "lr": 4.836291460296066e-07, "episode": 17168, "epoch": 0.8670707070707071, "step": 1073}, {"eps": 0, "objective/kl": 29.793800354003906, "objective/entropy": -17.03137969970703, "objective/non_score_reward": -1.4896900653839111, "objective/rlhf_reward": 15.447810173034668, "objective/scores": 17.0, "policy/approxkl_avg": 0.000868711736984551, "policy/clipfrac_avg": 0.0015887501649558544, "loss/policy_avg": 0.032950110733509064, "loss/value_avg": 0.5699096918106079, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17066001892089844, "val/ratio": 0.9895613193511963, "val/ratio_var": 7.112583261914551e-05, "val/num_eos_tokens": 84, "lr": 4.779142148665855e-07, "episode": 17184, "epoch": 0.8678787878787879, "step": 1074}, {"eps": 0, "objective/kl": 39.953025817871094, "objective/entropy": -46.34983825683594, "objective/non_score_reward": -1.997651219367981, "objective/rlhf_reward": 14.674223899841309, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008599478751420975, "policy/clipfrac_avg": 0.006935285869985819, "loss/policy_avg": -0.11098457127809525, "loss/value_avg": 0.5394589304924011, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17142105102539062, "val/ratio": 0.9890701770782471, "val/ratio_var": 3.540362376952544e-05, "val/num_eos_tokens": 73, "lr": 4.722315555370793e-07, "episode": 17200, "epoch": 0.8686868686868687, "step": 1075}, {"eps": 0, "objective/kl": 43.93402099609375, "objective/entropy": -62.084415435791016, "objective/non_score_reward": -2.1967010498046875, "objective/rlhf_reward": 12.162673950195312, "objective/scores": 14.375, "policy/approxkl_avg": 0.0009052088716998696, "policy/clipfrac_avg": 0.0032377864699810743, "loss/policy_avg": 0.11330074071884155, "loss/value_avg": 0.47666221857070923, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18408584594726562, "val/ratio": 0.9876794815063477, "val/ratio_var": 4.1342565964441746e-05, "val/num_eos_tokens": 588, "lr": 4.665812085953425e-07, "episode": 17216, "epoch": 0.8694949494949495, "step": 1076}, {"eps": 0, "objective/kl": 39.06366729736328, "objective/entropy": -73.24485778808594, "objective/non_score_reward": -1.9531835317611694, "objective/rlhf_reward": 13.5468168258667, "objective/scores": 15.5, "policy/approxkl_avg": 0.0009686506818979979, "policy/clipfrac_avg": 0.004011271521449089, "loss/policy_avg": -0.05885958671569824, "loss/value_avg": 0.9560117125511169, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18913555145263672, "val/ratio": 0.9870175719261169, "val/ratio_var": 8.096115197986364e-05, "val/num_eos_tokens": 200, "lr": 4.6096321436504e-07, "episode": 17232, "epoch": 0.8703030303030304, "step": 1077}, {"eps": 0, "objective/kl": 37.137413024902344, "objective/entropy": -116.47470092773438, "objective/non_score_reward": -1.8568708896636963, "objective/rlhf_reward": 14.861879348754883, "objective/scores": 16.75, "policy/approxkl_avg": 0.000727109843865037, "policy/clipfrac_avg": 0.002056294586509466, "loss/policy_avg": 0.18471674621105194, "loss/value_avg": 0.37227392196655273, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1612076759338379, "val/ratio": 0.9902770519256592, "val/ratio_var": 6.446443876484409e-05, "val/num_eos_tokens": 178, "lr": 4.5537761293894535e-07, "episode": 17248, "epoch": 0.8711111111111111, "step": 1078}, {"eps": 0, "objective/kl": 37.85641098022461, "objective/entropy": -93.82952880859375, "objective/non_score_reward": -1.8928205966949463, "objective/rlhf_reward": 13.458742141723633, "objective/scores": 15.375, "policy/approxkl_avg": 0.0009164538932964206, "policy/clipfrac_avg": 0.0050585465505719185, "loss/policy_avg": -0.16059936583042145, "loss/value_avg": 0.507465124130249, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15972977876663208, "val/ratio": 0.989949643611908, "val/ratio_var": 8.05418094387278e-05, "val/num_eos_tokens": 44, "lr": 4.4982444417866753e-07, "episode": 17264, "epoch": 0.8719191919191919, "step": 1079}, {"eps": 0, "objective/kl": 36.883766174316406, "objective/entropy": -93.29814910888672, "objective/non_score_reward": -1.8441884517669678, "objective/rlhf_reward": 13.952686309814453, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0010746137704700232, "policy/clipfrac_avg": 0.005830212030559778, "loss/policy_avg": 0.024168267846107483, "loss/value_avg": 0.5276412963867188, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1792011260986328, "val/ratio": 0.989915132522583, "val/ratio_var": 4.745582918985747e-05, "val/num_eos_tokens": 519, "lr": 4.4430374771435245e-07, "episode": 17280, "epoch": 0.8727272727272727, "step": 1080}, {"eps": 0, "objective/kl": 42.45173645019531, "objective/entropy": -163.2279815673828, "objective/non_score_reward": -2.122586727142334, "objective/rlhf_reward": 13.236787796020508, "objective/scores": 15.375, "policy/approxkl_avg": 0.0007963902316987514, "policy/clipfrac_avg": 0.0037931441329419613, "loss/policy_avg": -0.09750235080718994, "loss/value_avg": 0.4400123357772827, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17099809646606445, "val/ratio": 0.9886674880981445, "val/ratio_var": 8.438028453383595e-05, "val/num_eos_tokens": 294, "lr": 4.3881556294441516e-07, "episode": 17296, "epoch": 0.8735353535353535, "step": 1081}, {"eps": 0, "objective/kl": 42.34510803222656, "objective/entropy": -63.06423568725586, "objective/non_score_reward": -2.117255210876465, "objective/rlhf_reward": 14.288995742797852, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009070255910046399, "policy/clipfrac_avg": 0.0022676163353025913, "loss/policy_avg": -0.0659959465265274, "loss/value_avg": 0.2975969910621643, "val/clipfrac_avg": 0.0035635964013636112, "policy/entropy_avg": 0.18258285522460938, "val/ratio": 0.9886539578437805, "val/ratio_var": 6.365034641930833e-05, "val/num_eos_tokens": 576, "lr": 4.333599290352458e-07, "episode": 17312, "epoch": 0.8743434343434343, "step": 1082}, {"eps": 0, "objective/kl": 34.96723937988281, "objective/entropy": -124.63575744628906, "objective/non_score_reward": -1.7483620643615723, "objective/rlhf_reward": 15.900075912475586, "objective/scores": 17.625, "policy/approxkl_avg": 0.0008090031915344298, "policy/clipfrac_avg": 0.0031590727157890797, "loss/policy_avg": -0.12663836777210236, "loss/value_avg": 1.6143395900726318, "val/clipfrac_avg": 0.02003205195069313, "policy/entropy_avg": 0.16525745391845703, "val/ratio": 0.9890646934509277, "val/ratio_var": 7.246930908877403e-05, "val/num_eos_tokens": 359, "lr": 4.279368849209381e-07, "episode": 17328, "epoch": 0.8751515151515151, "step": 1083}, {"eps": 0, "objective/kl": 44.472023010253906, "objective/entropy": -55.43729019165039, "objective/non_score_reward": -2.2236011028289795, "objective/rlhf_reward": 13.721712112426758, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0009467223426327109, "policy/clipfrac_avg": 0.0063713230192661285, "loss/policy_avg": 0.10138820856809616, "loss/value_avg": 0.3533989191055298, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1949615478515625, "val/ratio": 0.986469030380249, "val/ratio_var": 5.372995656216517e-05, "val/num_eos_tokens": 42, "lr": 4.2254646930300845e-07, "episode": 17344, "epoch": 0.8759595959595959, "step": 1084}, {"eps": 0, "objective/kl": 35.82070541381836, "objective/entropy": -80.793701171875, "objective/non_score_reward": -1.7910352945327759, "objective/rlhf_reward": 13.552715301513672, "objective/scores": 15.375, "policy/approxkl_avg": 0.0007973198080435395, "policy/clipfrac_avg": 0.003301304765045643, "loss/policy_avg": -0.16358020901679993, "loss/value_avg": 0.7672687768936157, "val/clipfrac_avg": 0.0021186440717428923, "policy/entropy_avg": 0.15000247955322266, "val/ratio": 0.9896234273910522, "val/ratio_var": 5.3207288146950305e-05, "val/num_eos_tokens": 403, "lr": 4.171887206501191e-07, "episode": 17360, "epoch": 0.8767676767676768, "step": 1085}, {"eps": 0, "objective/kl": 33.1654167175293, "objective/entropy": -46.82775115966797, "objective/non_score_reward": -1.6582709550857544, "objective/rlhf_reward": 14.404230117797852, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008230544626712799, "policy/clipfrac_avg": 0.0030164497438818216, "loss/policy_avg": 0.20687676966190338, "loss/value_avg": 1.3663004636764526, "val/clipfrac_avg": 0.0026466259732842445, "policy/entropy_avg": 0.1555004119873047, "val/ratio": 0.9892673492431641, "val/ratio_var": 3.7006568163633347e-05, "val/num_eos_tokens": 47, "lr": 4.1186367719780737e-07, "episode": 17376, "epoch": 0.8775757575757576, "step": 1086}, {"eps": 0, "objective/kl": 45.35851287841797, "objective/entropy": -66.93091583251953, "objective/non_score_reward": -2.26792573928833, "objective/rlhf_reward": 14.122699737548828, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008988711051642895, "policy/clipfrac_avg": 0.004365711007267237, "loss/policy_avg": 0.18984848260879517, "loss/value_avg": 0.31801366806030273, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17530107498168945, "val/ratio": 0.9879393577575684, "val/ratio_var": 6.273078179219738e-05, "val/num_eos_tokens": 363, "lr": 4.0657137694820826e-07, "episode": 17392, "epoch": 0.8783838383838384, "step": 1087}, {"eps": 0, "objective/kl": 44.89594650268555, "objective/entropy": -96.627197265625, "objective/non_score_reward": -2.244797468185425, "objective/rlhf_reward": 14.028639793395996, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009503285982646048, "policy/clipfrac_avg": 0.004713431932032108, "loss/policy_avg": 0.14154401421546936, "loss/value_avg": 0.296213299036026, "val/clipfrac_avg": 0.004036702681332827, "policy/entropy_avg": 0.19094181060791016, "val/ratio": 0.9875110983848572, "val/ratio_var": 3.645908509497531e-05, "val/num_eos_tokens": 215, "lr": 4.013118576697844e-07, "episode": 17408, "epoch": 0.8791919191919192, "step": 1088}, {"eps": 0, "objective/kl": 44.09088897705078, "objective/entropy": -81.71890258789062, "objective/non_score_reward": -2.2045445442199707, "objective/rlhf_reward": 14.311080932617188, "objective/scores": 16.5, "policy/approxkl_avg": 0.0008286006050184369, "policy/clipfrac_avg": 0.0029842001385986805, "loss/policy_avg": -0.2717769145965576, "loss/value_avg": 0.6515541076660156, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16189241409301758, "val/ratio": 0.9892988204956055, "val/ratio_var": 5.128037810209207e-05, "val/num_eos_tokens": 391, "lr": 3.960851568970586e-07, "episode": 17424, "epoch": 0.88, "step": 1089}, {"eps": 0, "objective/kl": 45.33392333984375, "objective/entropy": -103.39527893066406, "objective/non_score_reward": -2.2666964530944824, "objective/rlhf_reward": 15.030179023742676, "objective/scores": 17.25, "policy/approxkl_avg": 0.0008385264663957059, "policy/clipfrac_avg": 0.0022599862422794104, "loss/policy_avg": -0.09007427841424942, "loss/value_avg": 0.174169659614563, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.167741060256958, "val/ratio": 0.9883161783218384, "val/ratio_var": 4.988587534171529e-05, "val/num_eos_tokens": 96, "lr": 3.908913119303448e-07, "episode": 17440, "epoch": 0.8808080808080808, "step": 1090}, {"eps": 0, "objective/kl": 38.23064422607422, "objective/entropy": -82.11065673828125, "objective/non_score_reward": -1.9115324020385742, "objective/rlhf_reward": 13.299405097961426, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0008716207230463624, "policy/clipfrac_avg": 0.0072744423523545265, "loss/policy_avg": 0.29356396198272705, "loss/value_avg": 0.8743624687194824, "val/clipfrac_avg": 0.0008928571478463709, "policy/entropy_avg": 0.16114044189453125, "val/ratio": 0.9882928133010864, "val/ratio_var": 6.514011329272762e-05, "val/num_eos_tokens": 91, "lr": 3.8573035983548167e-07, "episode": 17456, "epoch": 0.8816161616161616, "step": 1091}, {"eps": 0, "objective/kl": 37.99050521850586, "objective/entropy": -12.753183364868164, "objective/non_score_reward": -1.8995252847671509, "objective/rlhf_reward": 12.65125560760498, "objective/scores": 14.5625, "policy/approxkl_avg": 0.0010071105789393187, "policy/clipfrac_avg": 0.004005424678325653, "loss/policy_avg": 0.02994067780673504, "loss/value_avg": 0.792471170425415, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19425201416015625, "val/ratio": 0.9855411052703857, "val/ratio_var": 4.6690693125128746e-05, "val/num_eos_tokens": 14, "lr": 3.8060233744356634e-07, "episode": 17472, "epoch": 0.8824242424242424, "step": 1092}, {"eps": 0, "objective/kl": 48.05560302734375, "objective/entropy": -36.16312789916992, "objective/non_score_reward": -2.402780532836914, "objective/rlhf_reward": 14.300344467163086, "objective/scores": 16.75, "policy/approxkl_avg": 0.000979696400463581, "policy/clipfrac_avg": 0.005794232711195946, "loss/policy_avg": 0.1304478645324707, "loss/value_avg": 0.3748246431350708, "val/clipfrac_avg": 0.0026041667442768812, "policy/entropy_avg": 0.19660568237304688, "val/ratio": 0.9865772128105164, "val/ratio_var": 5.429132215795107e-05, "val/num_eos_tokens": 66, "lr": 3.7550728135069545e-07, "episode": 17488, "epoch": 0.8832323232323233, "step": 1093}, {"eps": 0, "objective/kl": 45.28865051269531, "objective/entropy": -71.43814086914062, "objective/non_score_reward": -2.264432668685913, "objective/rlhf_reward": 13.727754592895508, "objective/scores": 16.0, "policy/approxkl_avg": 0.000950466375797987, "policy/clipfrac_avg": 0.0027152306865900755, "loss/policy_avg": 0.07484120875597, "loss/value_avg": 0.2602160573005676, "val/clipfrac_avg": 0.0006720430101267993, "policy/entropy_avg": 0.20662879943847656, "val/ratio": 0.9866384863853455, "val/ratio_var": 7.357261347351596e-05, "val/num_eos_tokens": 494, "lr": 3.704452279177012e-07, "episode": 17504, "epoch": 0.8840404040404041, "step": 1094}, {"eps": 0, "objective/kl": 39.92009353637695, "objective/entropy": -48.0823974609375, "objective/non_score_reward": -1.9960048198699951, "objective/rlhf_reward": 12.980557441711426, "objective/scores": 15.0, "policy/approxkl_avg": 0.0010038036853075027, "policy/clipfrac_avg": 0.004399286117404699, "loss/policy_avg": 0.10963790118694305, "loss/value_avg": 0.7320235967636108, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19763946533203125, "val/ratio": 0.987025260925293, "val/ratio_var": 3.8341207982739434e-05, "val/num_eos_tokens": 436, "lr": 3.6541621326989183e-07, "episode": 17520, "epoch": 0.8848484848484849, "step": 1095}, {"eps": 0, "objective/kl": 41.74889373779297, "objective/entropy": -51.004615783691406, "objective/non_score_reward": -2.087444543838501, "objective/rlhf_reward": 14.545368194580078, "objective/scores": 16.625, "policy/approxkl_avg": 0.0010824615601450205, "policy/clipfrac_avg": 0.00963993463665247, "loss/policy_avg": 0.05299535021185875, "loss/value_avg": 0.2717387080192566, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20147705078125, "val/ratio": 0.9875917434692383, "val/ratio_var": 6.199166091391817e-05, "val/num_eos_tokens": 345, "lr": 3.6042027329679487e-07, "episode": 17536, "epoch": 0.8856565656565657, "step": 1096}, {"eps": 0, "objective/kl": 41.82084655761719, "objective/entropy": -108.7376937866211, "objective/non_score_reward": -2.0910425186157227, "objective/rlhf_reward": 14.158958435058594, "objective/scores": 16.25, "policy/approxkl_avg": 0.000758812646381557, "policy/clipfrac_avg": 0.003666321514174342, "loss/policy_avg": -0.08447069674730301, "loss/value_avg": 0.38099169731140137, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15442466735839844, "val/ratio": 0.9902445077896118, "val/ratio_var": 2.8433618354029022e-05, "val/num_eos_tokens": 244, "lr": 3.5545744365190204e-07, "episode": 17552, "epoch": 0.8864646464646465, "step": 1097}, {"eps": 0, "objective/kl": 33.65018081665039, "objective/entropy": -131.56527709960938, "objective/non_score_reward": -1.682509183883667, "objective/rlhf_reward": 13.74717903137207, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0009085559286177158, "policy/clipfrac_avg": 0.004985325504094362, "loss/policy_avg": -0.47139230370521545, "loss/value_avg": 1.4234567880630493, "val/clipfrac_avg": 0.00024242099607363343, "policy/entropy_avg": 0.16993939876556396, "val/ratio": 0.9888011813163757, "val/ratio_var": 7.847072265576571e-05, "val/num_eos_tokens": 1220, "lr": 3.5052775975241203e-07, "episode": 17568, "epoch": 0.8872727272727273, "step": 1098}, {"eps": 0, "objective/kl": 37.358299255371094, "objective/entropy": -42.34114074707031, "objective/non_score_reward": -1.867915153503418, "objective/rlhf_reward": 14.428959846496582, "objective/scores": 16.25, "policy/approxkl_avg": 0.001106644980609417, "policy/clipfrac_avg": 0.0073922015726566315, "loss/policy_avg": 0.16145119071006775, "loss/value_avg": 0.5360366106033325, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.216278076171875, "val/ratio": 0.9858206510543823, "val/ratio_var": 6.1017286498099566e-05, "val/num_eos_tokens": 172, "lr": 3.4563125677897936e-07, "episode": 17584, "epoch": 0.888080808080808, "step": 1099}, {"eps": 0, "objective/kl": 43.24127197265625, "objective/entropy": -32.0341682434082, "objective/non_score_reward": -2.1620638370513916, "objective/rlhf_reward": 12.939498901367188, "objective/scores": 15.125, "policy/approxkl_avg": 0.0010629122843965888, "policy/clipfrac_avg": 0.003724365960806608, "loss/policy_avg": 0.043675780296325684, "loss/value_avg": 0.5687451362609863, "val/clipfrac_avg": 0.005150945391505957, "policy/entropy_avg": 0.20337867736816406, "val/ratio": 0.9857684969902039, "val/ratio_var": 4.0027771319728345e-05, "val/num_eos_tokens": 106, "lr": 3.4076796967546176e-07, "episode": 17600, "epoch": 0.8888888888888888, "step": 1100}, {"eps": 0, "objective/kl": 40.037052154541016, "objective/entropy": -65.23380279541016, "objective/non_score_reward": -2.001852512359619, "objective/rlhf_reward": 13.349709510803223, "objective/scores": 15.375, "policy/approxkl_avg": 0.0010041861096397042, "policy/clipfrac_avg": 0.004063514992594719, "loss/policy_avg": -0.0018694326281547546, "loss/value_avg": 0.43309682607650757, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19306087493896484, "val/ratio": 0.9867812395095825, "val/ratio_var": 5.167184644960798e-05, "val/num_eos_tokens": 14, "lr": 3.359379331486762e-07, "episode": 17616, "epoch": 0.8896969696969697, "step": 1101}, {"eps": 0, "objective/kl": 41.81010055541992, "objective/entropy": -81.65939331054688, "objective/non_score_reward": -2.0905051231384277, "objective/rlhf_reward": 14.393869400024414, "objective/scores": 16.5, "policy/approxkl_avg": 0.000925385276786983, "policy/clipfrac_avg": 0.004391741007566452, "loss/policy_avg": 0.048976313322782516, "loss/value_avg": 0.3672630190849304, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1823406219482422, "val/ratio": 0.9886959791183472, "val/ratio_var": 5.684143616235815e-05, "val/num_eos_tokens": 128, "lr": 3.311411816681426e-07, "episode": 17632, "epoch": 0.8905050505050505, "step": 1102}, {"eps": 0, "objective/kl": 39.38318634033203, "objective/entropy": -97.10613250732422, "objective/non_score_reward": -1.9691593647003174, "objective/rlhf_reward": 14.226152420043945, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008451448520645499, "policy/clipfrac_avg": 0.0035835830494761467, "loss/policy_avg": 0.02880220115184784, "loss/value_avg": 0.3456343114376068, "val/clipfrac_avg": 0.0004976114723831415, "policy/entropy_avg": 0.18262100219726562, "val/ratio": 0.9880388975143433, "val/ratio_var": 4.812321640201844e-05, "val/num_eos_tokens": 81, "lr": 3.263777494658449e-07, "episode": 17648, "epoch": 0.8913131313131313, "step": 1103}, {"eps": 0, "objective/kl": 39.81349182128906, "objective/entropy": -138.00392150878906, "objective/non_score_reward": -1.990674614906311, "objective/rlhf_reward": 13.14213752746582, "objective/scores": 15.125, "policy/approxkl_avg": 0.0009344244026578963, "policy/clipfrac_avg": 0.004199904855340719, "loss/policy_avg": -0.15117420256137848, "loss/value_avg": 1.1375534534454346, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1756887435913086, "val/ratio": 0.9869411587715149, "val/ratio_var": 8.181409793905914e-05, "val/num_eos_tokens": 173, "lr": 3.216476705359839e-07, "episode": 17664, "epoch": 0.8921212121212121, "step": 1104}, {"eps": 0, "objective/kl": 38.82806396484375, "objective/entropy": -191.18865966796875, "objective/non_score_reward": -1.9414031505584717, "objective/rlhf_reward": 13.941408157348633, "objective/scores": 15.875, "policy/approxkl_avg": 0.0005901164840906858, "policy/clipfrac_avg": 0.005352393724024296, "loss/policy_avg": -0.40621763467788696, "loss/value_avg": 0.46248072385787964, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1206204891204834, "val/ratio": 0.9924303293228149, "val/ratio_var": 4.216426896164194e-05, "val/num_eos_tokens": 236, "lr": 3.1695097863473355e-07, "episode": 17680, "epoch": 0.8929292929292929, "step": 1105}, {"eps": 0, "objective/kl": 37.186309814453125, "objective/entropy": -87.4638900756836, "objective/non_score_reward": -1.8593155145645142, "objective/rlhf_reward": 15.757871627807617, "objective/scores": 17.625, "policy/approxkl_avg": 0.0008562589064240456, "policy/clipfrac_avg": 0.0030250055715441704, "loss/policy_avg": -0.11594279855489731, "loss/value_avg": 0.7615197896957397, "val/clipfrac_avg": 8.561644062865525e-05, "policy/entropy_avg": 0.1770009994506836, "val/ratio": 0.9886528253555298, "val/ratio_var": 7.094601460266858e-05, "val/num_eos_tokens": 557, "lr": 3.122877072800046e-07, "episode": 17696, "epoch": 0.8937373737373737, "step": 1106}, {"eps": 0, "objective/kl": 39.78605651855469, "objective/entropy": -110.63394927978516, "objective/non_score_reward": -1.9893031120300293, "objective/rlhf_reward": 14.362258911132812, "objective/scores": 16.375, "policy/approxkl_avg": 0.0007805367931723595, "policy/clipfrac_avg": 0.0034728883765637875, "loss/policy_avg": -0.12702879309654236, "loss/value_avg": 0.3219323754310608, "val/clipfrac_avg": 0.00021701389050576836, "policy/entropy_avg": 0.14988577365875244, "val/ratio": 0.9894638657569885, "val/ratio_var": 6.243310053832829e-05, "val/num_eos_tokens": 344, "lr": 3.076578897511978e-07, "episode": 17712, "epoch": 0.8945454545454545, "step": 1107}, {"eps": 0, "objective/kl": 35.881317138671875, "objective/entropy": -74.73516845703125, "objective/non_score_reward": -1.7940657138824463, "objective/rlhf_reward": 14.002809524536133, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0009087615762837231, "policy/clipfrac_avg": 0.004955953918397427, "loss/policy_avg": -0.030154824256896973, "loss/value_avg": 0.6115067005157471, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1930408477783203, "val/ratio": 0.9869700074195862, "val/ratio_var": 6.0412487073335797e-05, "val/num_eos_tokens": 300, "lr": 3.0306155908897626e-07, "episode": 17728, "epoch": 0.8953535353535353, "step": 1108}, {"eps": 0, "objective/kl": 33.56185531616211, "objective/entropy": -105.74530029296875, "objective/non_score_reward": -1.6780929565429688, "objective/rlhf_reward": 12.939094543457031, "objective/scores": 14.625, "policy/approxkl_avg": 0.0009151956764981151, "policy/clipfrac_avg": 0.004283002577722073, "loss/policy_avg": -0.01582721620798111, "loss/value_avg": 0.5954544544219971, "val/clipfrac_avg": 0.001953125, "policy/entropy_avg": 0.18004417419433594, "val/ratio": 0.9886264801025391, "val/ratio_var": 6.606351962545887e-05, "val/num_eos_tokens": 107, "lr": 2.984987480950191e-07, "episode": 17744, "epoch": 0.8961616161616162, "step": 1109}, {"eps": 0, "objective/kl": 40.10255432128906, "objective/entropy": -105.83699798583984, "objective/non_score_reward": -2.0051279067993164, "objective/rlhf_reward": 13.768309593200684, "objective/scores": 15.75, "policy/approxkl_avg": 0.0009933009278029203, "policy/clipfrac_avg": 0.002607613569125533, "loss/policy_avg": -0.0613558366894722, "loss/value_avg": 0.37346547842025757, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19908523559570312, "val/ratio": 0.9867061376571655, "val/ratio_var": 8.13972728792578e-05, "val/num_eos_tokens": 189, "lr": 2.939694893317979e-07, "episode": 17760, "epoch": 0.896969696969697, "step": 1110}, {"eps": 0, "objective/kl": 45.03960418701172, "objective/entropy": -69.66899108886719, "objective/non_score_reward": -2.2519803047180176, "objective/rlhf_reward": 15.544894218444824, "objective/scores": 17.75, "policy/approxkl_avg": 0.0010472675785422325, "policy/clipfrac_avg": 0.0032264026813209057, "loss/policy_avg": 0.0873388946056366, "loss/value_avg": 0.6430274844169617, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20671892166137695, "val/ratio": 0.9859870672225952, "val/ratio_var": 6.778486567782238e-05, "val/num_eos_tokens": 714, "lr": 2.894738151223331e-07, "episode": 17776, "epoch": 0.8977777777777778, "step": 1111}, {"eps": 0, "objective/kl": 39.79743957519531, "objective/entropy": -89.71099090576172, "objective/non_score_reward": -1.9898720979690552, "objective/rlhf_reward": 16.385128021240234, "objective/scores": 18.375, "policy/approxkl_avg": 0.00102079042699188, "policy/clipfrac_avg": 0.004375361371785402, "loss/policy_avg": -0.13560356199741364, "loss/value_avg": 0.6080023646354675, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19225120544433594, "val/ratio": 0.9872239828109741, "val/ratio_var": 7.309510692721233e-05, "val/num_eos_tokens": 296, "lr": 2.850117575499739e-07, "episode": 17792, "epoch": 0.8985858585858586, "step": 1112}, {"eps": 0, "objective/kl": 48.40119934082031, "objective/entropy": -31.377525329589844, "objective/non_score_reward": -2.4200599193573, "objective/rlhf_reward": 15.626814842224121, "objective/scores": 18.0, "policy/approxkl_avg": 0.0010773141402751207, "policy/clipfrac_avg": 0.007447387091815472, "loss/policy_avg": -0.33778393268585205, "loss/value_avg": 0.4939768612384796, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2159743309020996, "val/ratio": 0.9851824641227722, "val/ratio_var": 7.278792327269912e-05, "val/num_eos_tokens": 28, "lr": 2.8058334845816214e-07, "episode": 17808, "epoch": 0.8993939393939394, "step": 1113}, {"eps": 0, "objective/kl": 37.62234115600586, "objective/entropy": -30.16031265258789, "objective/non_score_reward": -1.8811171054840088, "objective/rlhf_reward": 12.79857063293457, "objective/scores": 14.6875, "policy/approxkl_avg": 0.0010548201389610767, "policy/clipfrac_avg": 0.0043435124680399895, "loss/policy_avg": -0.1028350442647934, "loss/value_avg": 0.5992170572280884, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19416046142578125, "val/ratio": 0.9875701665878296, "val/ratio_var": 4.415150760905817e-05, "val/num_eos_tokens": 115, "lr": 2.761886194502089e-07, "episode": 17824, "epoch": 0.9002020202020202, "step": 1114}, {"eps": 0, "objective/kl": 44.19203567504883, "objective/entropy": -98.54946899414062, "objective/non_score_reward": -2.209601879119873, "objective/rlhf_reward": 12.516960144042969, "objective/scores": 14.75, "policy/approxkl_avg": 0.0008550393395125866, "policy/clipfrac_avg": 0.0035258647985756397, "loss/policy_avg": 0.15831202268600464, "loss/value_avg": 0.5015832185745239, "val/clipfrac_avg": 9.245562250725925e-05, "policy/entropy_avg": 0.17531871795654297, "val/ratio": 0.9883965253829956, "val/ratio_var": 6.70754598104395e-05, "val/num_eos_tokens": 277, "lr": 2.71827601889067e-07, "episode": 17840, "epoch": 0.901010101010101, "step": 1115}, {"eps": 0, "objective/kl": 40.806068420410156, "objective/entropy": -89.13906860351562, "objective/non_score_reward": -2.0403032302856445, "objective/rlhf_reward": 14.842509269714355, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009749368182383478, "policy/clipfrac_avg": 0.004587238188832998, "loss/policy_avg": -0.0883672684431076, "loss/value_avg": 0.38530808687210083, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1862497329711914, "val/ratio": 0.9878228902816772, "val/ratio_var": 6.971527182031423e-05, "val/num_eos_tokens": 42, "lr": 2.6750032689710604e-07, "episode": 17856, "epoch": 0.9018181818181819, "step": 1116}, {"eps": 0, "objective/kl": 39.73638916015625, "objective/entropy": -94.74272155761719, "objective/non_score_reward": -1.9868193864822388, "objective/rlhf_reward": 14.48974323272705, "objective/scores": 16.5, "policy/approxkl_avg": 0.001014375826343894, "policy/clipfrac_avg": 0.009114467538893223, "loss/policy_avg": 0.35861504077911377, "loss/value_avg": 0.47818782925605774, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18755674362182617, "val/ratio": 0.9870187044143677, "val/ratio_var": 6.580681656487286e-05, "val/num_eos_tokens": 316, "lr": 2.6320682535589557e-07, "episode": 17872, "epoch": 0.9026262626262627, "step": 1117}, {"eps": 0, "objective/kl": 38.03620147705078, "objective/entropy": -60.80832290649414, "objective/non_score_reward": -1.9018102884292603, "objective/rlhf_reward": 14.418501853942871, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009664712124504149, "policy/clipfrac_avg": 0.0054984488524496555, "loss/policy_avg": 0.03886333107948303, "loss/value_avg": 0.5049983263015747, "val/clipfrac_avg": 0.00028935185400769114, "policy/entropy_avg": 0.18600749969482422, "val/ratio": 0.9878861904144287, "val/ratio_var": 8.505670120939612e-05, "val/num_eos_tokens": 538, "lr": 2.589471279059791e-07, "episode": 17888, "epoch": 0.9034343434343435, "step": 1118}, {"eps": 0, "objective/kl": 40.48139190673828, "objective/entropy": -66.79815673828125, "objective/non_score_reward": -2.0240697860717773, "objective/rlhf_reward": 13.483742713928223, "objective/scores": 15.5, "policy/approxkl_avg": 0.0010647093877196312, "policy/clipfrac_avg": 0.005042470060288906, "loss/policy_avg": -0.30915409326553345, "loss/value_avg": 0.6310875415802002, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2105545997619629, "val/ratio": 0.9860644340515137, "val/ratio_var": 7.680879207327962e-05, "val/num_eos_tokens": 341, "lr": 2.547212649466568e-07, "episode": 17904, "epoch": 0.9042424242424243, "step": 1119}, {"eps": 0, "objective/kl": 37.41370391845703, "objective/entropy": -109.19269561767578, "objective/non_score_reward": -1.8706854581832886, "objective/rlhf_reward": 15.559001922607422, "objective/scores": 17.375, "policy/approxkl_avg": 0.0008096316596493125, "policy/clipfrac_avg": 0.007008845917880535, "loss/policy_avg": -0.0354868620634079, "loss/value_avg": 0.43115413188934326, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1526775360107422, "val/ratio": 0.990079402923584, "val/ratio_var": 6.870418292237446e-05, "val/num_eos_tokens": 27, "lr": 2.5052926663577006e-07, "episode": 17920, "epoch": 0.9050505050505051, "step": 1120}, {"eps": 0, "objective/kl": 34.01490020751953, "objective/entropy": -81.66447448730469, "objective/non_score_reward": -1.7007449865341187, "objective/rlhf_reward": 14.06488037109375, "objective/scores": 15.75, "policy/approxkl_avg": 0.0011045493884012103, "policy/clipfrac_avg": 0.004975738935172558, "loss/policy_avg": -0.10384486615657806, "loss/value_avg": 0.5555015802383423, "val/clipfrac_avg": 0.00020559210679493845, "policy/entropy_avg": 0.19969654083251953, "val/ratio": 0.9861068725585938, "val/ratio_var": 0.0001140525855589658, "val/num_eos_tokens": 424, "lr": 2.4637116288948606e-07, "episode": 17936, "epoch": 0.9058585858585858, "step": 1121}, {"eps": 0, "objective/kl": 36.03460693359375, "objective/entropy": -51.88709259033203, "objective/non_score_reward": -1.8017305135726929, "objective/rlhf_reward": 13.284207344055176, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0007865545339882374, "policy/clipfrac_avg": 0.005190032534301281, "loss/policy_avg": 0.08499134331941605, "loss/value_avg": 0.4820760488510132, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1644287109375, "val/ratio": 0.9896594882011414, "val/ratio_var": 3.428427953622304e-05, "val/num_eos_tokens": 97, "lr": 2.4224698338208344e-07, "episode": 17952, "epoch": 0.9066666666666666, "step": 1122}, {"eps": 0, "objective/kl": 43.59098815917969, "objective/entropy": -86.18510437011719, "objective/non_score_reward": -2.179549217224121, "objective/rlhf_reward": 14.422013282775879, "objective/scores": 16.625, "policy/approxkl_avg": 0.000971743487752974, "policy/clipfrac_avg": 0.006423149257898331, "loss/policy_avg": -0.17406421899795532, "loss/value_avg": 0.5896267294883728, "val/clipfrac_avg": 0.0003633720916695893, "policy/entropy_avg": 0.18558311462402344, "val/ratio": 0.9873388409614563, "val/ratio_var": 6.847102486062795e-05, "val/num_eos_tokens": 28, "lr": 2.3815675754573885e-07, "episode": 17968, "epoch": 0.9074747474747474, "step": 1123}, {"eps": 0, "objective/kl": 46.467933654785156, "objective/entropy": -15.714594841003418, "objective/non_score_reward": -2.323396682739258, "objective/rlhf_reward": 13.957853317260742, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009007846238091588, "policy/clipfrac_avg": 0.004247956909239292, "loss/policy_avg": 0.003323908895254135, "loss/value_avg": 0.4784785807132721, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18125152587890625, "val/ratio": 0.9878742694854736, "val/ratio_var": 4.734889080282301e-05, "val/num_eos_tokens": 91, "lr": 2.3410051457032179e-07, "episode": 17984, "epoch": 0.9082828282828282, "step": 1124}, {"eps": 0, "objective/kl": 28.815372467041016, "objective/entropy": -78.62959289550781, "objective/non_score_reward": -1.4407685995101929, "objective/rlhf_reward": 13.76235580444336, "objective/scores": 15.1875, "policy/approxkl_avg": 0.0007939373608678579, "policy/clipfrac_avg": 0.0023933430202305317, "loss/policy_avg": -0.34681111574172974, "loss/value_avg": 1.8761348724365234, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14340460300445557, "val/ratio": 0.9906671047210693, "val/ratio_var": 8.598795102443546e-05, "val/num_eos_tokens": 162, "lr": 2.3007828340318117e-07, "episode": 18000, "epoch": 0.9090909090909091, "step": 1125}, {"eps": 0, "objective/kl": 41.48188018798828, "objective/entropy": -84.69230651855469, "objective/non_score_reward": -2.07409405708313, "objective/rlhf_reward": 14.402467727661133, "objective/scores": 16.5, "policy/approxkl_avg": 0.0007599142845720053, "policy/clipfrac_avg": 0.004918557591736317, "loss/policy_avg": -0.10182956606149673, "loss/value_avg": 0.9262669682502747, "val/clipfrac_avg": 0.0003810975467786193, "policy/entropy_avg": 0.14319229125976562, "val/ratio": 0.9904208779335022, "val/ratio_var": 5.685169162461534e-05, "val/num_eos_tokens": 279, "lr": 2.2609009274894124e-07, "episode": 18016, "epoch": 0.9098989898989899, "step": 1126}, {"eps": 0, "objective/kl": 36.32238006591797, "objective/entropy": 6.487635612487793, "objective/non_score_reward": -1.8161190748214722, "objective/rlhf_reward": 12.402630805969238, "objective/scores": 14.25, "policy/approxkl_avg": 0.0012117292499169707, "policy/clipfrac_avg": 0.003925529308617115, "loss/policy_avg": -0.05897387117147446, "loss/value_avg": 0.6550663709640503, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.22541046142578125, "val/ratio": 0.9840712547302246, "val/ratio_var": 4.843049100600183e-05, "val/num_eos_tokens": 48, "lr": 2.2213597106929608e-07, "episode": 18032, "epoch": 0.9107070707070707, "step": 1127}, {"eps": 0, "objective/kl": 43.229087829589844, "objective/entropy": -81.51445007324219, "objective/non_score_reward": -2.161454439163208, "objective/rlhf_reward": 14.604171752929688, "objective/scores": 16.75, "policy/approxkl_avg": 0.000821998284664005, "policy/clipfrac_avg": 0.003186737420037389, "loss/policy_avg": -0.2118905931711197, "loss/value_avg": 0.2701466381549835, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16611194610595703, "val/ratio": 0.9885213971138, "val/ratio_var": 5.360436989576556e-05, "val/num_eos_tokens": 379, "lr": 2.1821594658280932e-07, "episode": 18048, "epoch": 0.9115151515151515, "step": 1128}, {"eps": 0, "objective/kl": 43.020301818847656, "objective/entropy": -90.69831085205078, "objective/non_score_reward": -2.151015281677246, "objective/rlhf_reward": 15.84117317199707, "objective/scores": 18.0, "policy/approxkl_avg": 0.0008969155605882406, "policy/clipfrac_avg": 0.004066394176334143, "loss/policy_avg": 0.07222969084978104, "loss/value_avg": 0.13441604375839233, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18462800979614258, "val/ratio": 0.9881325364112854, "val/ratio_var": 5.7613899116404355e-05, "val/num_eos_tokens": 122, "lr": 2.1433004726470695e-07, "episode": 18064, "epoch": 0.9123232323232323, "step": 1129}, {"eps": 0, "objective/kl": 39.517765045166016, "objective/entropy": -97.6414566040039, "objective/non_score_reward": -1.9758882522583008, "objective/rlhf_reward": 13.8444242477417, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0008600642322562635, "policy/clipfrac_avg": 0.002656108234077692, "loss/policy_avg": 0.030582871288061142, "loss/value_avg": 0.32031866908073425, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16113853454589844, "val/ratio": 0.9883853793144226, "val/ratio_var": 6.297288200585172e-05, "val/num_eos_tokens": 730, "lr": 2.1047830084668264e-07, "episode": 18080, "epoch": 0.9131313131313131, "step": 1130}, {"eps": 0, "objective/kl": 34.70388412475586, "objective/entropy": -67.81413269042969, "objective/non_score_reward": -1.735194206237793, "objective/rlhf_reward": 14.077305793762207, "objective/scores": 15.8125, "policy/approxkl_avg": 0.0009027072228491306, "policy/clipfrac_avg": 0.004612314514815807, "loss/policy_avg": 0.004439752548933029, "loss/value_avg": 0.5337626934051514, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18857192993164062, "val/ratio": 0.9875516891479492, "val/ratio_var": 7.778128201607615e-05, "val/num_eos_tokens": 89, "lr": 2.0666073481669714e-07, "episode": 18096, "epoch": 0.9139393939393939, "step": 1131}, {"eps": 0, "objective/kl": 41.89903259277344, "objective/entropy": -57.457305908203125, "objective/non_score_reward": -2.094951629638672, "objective/rlhf_reward": 15.764423370361328, "objective/scores": 17.875, "policy/approxkl_avg": 0.001014683861285448, "policy/clipfrac_avg": 0.005427089519798756, "loss/policy_avg": -0.0649208128452301, "loss/value_avg": 0.2573353052139282, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2117767333984375, "val/ratio": 0.985901415348053, "val/ratio_var": 5.110340498504229e-05, "val/num_eos_tokens": 81, "lr": 2.0287737641878358e-07, "episode": 18112, "epoch": 0.9147474747474748, "step": 1132}, {"eps": 0, "objective/kl": 47.3048095703125, "objective/entropy": -32.120155334472656, "objective/non_score_reward": -2.3652405738830566, "objective/rlhf_reward": 13.720697402954102, "objective/scores": 16.125, "policy/approxkl_avg": 0.0010292984079569578, "policy/clipfrac_avg": 0.004157859366387129, "loss/policy_avg": -0.03162732720375061, "loss/value_avg": 0.41586339473724365, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20333099365234375, "val/ratio": 0.9864168167114258, "val/ratio_var": 5.427391442935914e-05, "val/num_eos_tokens": 51, "lr": 1.991282526528543e-07, "episode": 18128, "epoch": 0.9155555555555556, "step": 1133}, {"eps": 0, "objective/kl": 41.02958679199219, "objective/entropy": -68.75173950195312, "objective/non_score_reward": -2.0514793395996094, "objective/rlhf_reward": 13.80789566040039, "objective/scores": 15.875, "policy/approxkl_avg": 0.0009240241488441825, "policy/clipfrac_avg": 0.0033770925365388393, "loss/policy_avg": 0.0864434763789177, "loss/value_avg": 0.2746288478374481, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.182037353515625, "val/ratio": 0.9879450798034668, "val/ratio_var": 5.012234760215506e-05, "val/num_eos_tokens": 236, "lr": 1.9541339027450256e-07, "episode": 18144, "epoch": 0.9163636363636364, "step": 1134}, {"eps": 0, "objective/kl": 38.17439270019531, "objective/entropy": -23.645580291748047, "objective/non_score_reward": -1.9087196588516235, "objective/rlhf_reward": 15.474092483520508, "objective/scores": 17.375, "policy/approxkl_avg": 0.000994181027635932, "policy/clipfrac_avg": 0.0035517350770533085, "loss/policy_avg": 0.06688633561134338, "loss/value_avg": 0.4891657829284668, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21059799194335938, "val/ratio": 0.9864646196365356, "val/ratio_var": 6.11128198215738e-05, "val/num_eos_tokens": 4, "lr": 1.9173281579481896e-07, "episode": 18160, "epoch": 0.9171717171717172, "step": 1135}, {"eps": 0, "objective/kl": 43.086509704589844, "objective/entropy": -43.90042495727539, "objective/non_score_reward": -2.154325485229492, "objective/rlhf_reward": 14.970674514770508, "objective/scores": 17.125, "policy/approxkl_avg": 0.0010021792259067297, "policy/clipfrac_avg": 0.00640224339440465, "loss/policy_avg": 0.09911377727985382, "loss/value_avg": 0.18598923087120056, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19201087951660156, "val/ratio": 0.9868404865264893, "val/ratio_var": 3.5931021557189524e-05, "val/num_eos_tokens": 455, "lr": 1.8808655548019638e-07, "episode": 18176, "epoch": 0.917979797979798, "step": 1136}, {"eps": 0, "objective/kl": 41.71107482910156, "objective/entropy": -149.0535125732422, "objective/non_score_reward": -2.0855536460876465, "objective/rlhf_reward": 15.117570877075195, "objective/scores": 17.25, "policy/approxkl_avg": 0.0006844973540864885, "policy/clipfrac_avg": 0.004837099928408861, "loss/policy_avg": -0.0433524064719677, "loss/value_avg": 0.6105636358261108, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1427140235900879, "val/ratio": 0.990210235118866, "val/ratio_var": 5.7032579206861556e-05, "val/num_eos_tokens": 356, "lr": 1.8447463535214872e-07, "episode": 18192, "epoch": 0.9187878787878788, "step": 1137}, {"eps": 0, "objective/kl": 38.607025146484375, "objective/entropy": -55.38002395629883, "objective/non_score_reward": -1.9303512573242188, "objective/rlhf_reward": 14.397773742675781, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009189241100102663, "policy/clipfrac_avg": 0.003879917087033391, "loss/policy_avg": -0.038881611078977585, "loss/value_avg": 0.2666724920272827, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15883827209472656, "val/ratio": 0.9893890619277954, "val/ratio_var": 5.718938700738363e-05, "val/num_eos_tokens": 41, "lr": 1.8089708118711525e-07, "episode": 18208, "epoch": 0.9195959595959596, "step": 1138}, {"eps": 0, "objective/kl": 34.568389892578125, "objective/entropy": -44.06977462768555, "objective/non_score_reward": -1.728419542312622, "objective/rlhf_reward": 14.62314224243164, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008048653253354132, "policy/clipfrac_avg": 0.006366053596138954, "loss/policy_avg": 0.06429219990968704, "loss/value_avg": 0.8066933155059814, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18921661376953125, "val/ratio": 0.9889757037162781, "val/ratio_var": 3.707074938574806e-05, "val/num_eos_tokens": 33, "lr": 1.7735391851628814e-07, "episode": 18224, "epoch": 0.9204040404040404, "step": 1139}, {"eps": 0, "objective/kl": 39.761756896972656, "objective/entropy": -85.9615707397461, "objective/non_score_reward": -1.988087773323059, "objective/rlhf_reward": 14.47284984588623, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009051311062648892, "policy/clipfrac_avg": 0.0030989747028797865, "loss/policy_avg": 0.14161835610866547, "loss/value_avg": 0.728713870048523, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15839195251464844, "val/ratio": 0.9888681173324585, "val/ratio_var": 7.059999916236848e-05, "val/num_eos_tokens": 59, "lr": 1.7384517262542255e-07, "episode": 18240, "epoch": 0.9212121212121213, "step": 1140}, {"eps": 0, "objective/kl": 42.769935607910156, "objective/entropy": -50.15681457519531, "objective/non_score_reward": -2.1384968757629395, "objective/rlhf_reward": 14.103690147399902, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009481002343818545, "policy/clipfrac_avg": 0.0051858918741345406, "loss/policy_avg": 0.10231739282608032, "loss/value_avg": 0.7710623145103455, "val/clipfrac_avg": 2.8775322789442725e-05, "policy/entropy_avg": 0.2104654312133789, "val/ratio": 0.9877794981002808, "val/ratio_var": 6.828846380813047e-05, "val/num_eos_tokens": 158, "lr": 1.7037086855465902e-07, "episode": 18256, "epoch": 0.9220202020202021, "step": 1141}, {"eps": 0, "objective/kl": 37.26749801635742, "objective/entropy": -83.49034118652344, "objective/non_score_reward": -1.863374948501587, "objective/rlhf_reward": 13.496000289916992, "objective/scores": 15.375, "policy/approxkl_avg": 0.0008203347679227591, "policy/clipfrac_avg": 0.007063254248350859, "loss/policy_avg": 0.06746610999107361, "loss/value_avg": 0.32568371295928955, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16310882568359375, "val/ratio": 0.9898014068603516, "val/ratio_var": 4.5457767555490136e-05, "val/num_eos_tokens": 109, "lr": 1.6693103109834408e-07, "episode": 18272, "epoch": 0.9228282828282828, "step": 1142}, {"eps": 0, "objective/kl": 45.40270233154297, "objective/entropy": -22.768522262573242, "objective/non_score_reward": -2.2701351642608643, "objective/rlhf_reward": 14.222051620483398, "objective/scores": 16.5, "policy/approxkl_avg": 0.001096026157028973, "policy/clipfrac_avg": 0.004881397821009159, "loss/policy_avg": -0.1702660322189331, "loss/value_avg": 0.35972562432289124, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21602249145507812, "val/ratio": 0.9855197668075562, "val/ratio_var": 3.710871169460006e-05, "val/num_eos_tokens": 105, "lr": 1.6352568480485277e-07, "episode": 18288, "epoch": 0.9236363636363636, "step": 1143}, {"eps": 0, "objective/kl": 33.057132720947266, "objective/entropy": -49.97838592529297, "objective/non_score_reward": -1.6528568267822266, "objective/rlhf_reward": 15.800268173217773, "objective/scores": 17.5, "policy/approxkl_avg": 0.0009639778290875256, "policy/clipfrac_avg": 0.00445273332297802, "loss/policy_avg": 0.19920386373996735, "loss/value_avg": 0.5300213098526001, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.180816650390625, "val/ratio": 0.9873702526092529, "val/ratio_var": 4.910973075311631e-05, "val/num_eos_tokens": 508, "lr": 1.6015485397641696e-07, "episode": 18304, "epoch": 0.9244444444444444, "step": 1144}, {"eps": 0, "objective/kl": 46.23842239379883, "objective/entropy": -32.52317810058594, "objective/non_score_reward": -2.3119211196899414, "objective/rlhf_reward": 14.336517333984375, "objective/scores": 16.625, "policy/approxkl_avg": 0.0009981514886021614, "policy/clipfrac_avg": 0.002960982732474804, "loss/policy_avg": -0.09338352829217911, "loss/value_avg": 0.23753158748149872, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20490646362304688, "val/ratio": 0.9861640334129333, "val/ratio_var": 5.2834289817838e-05, "val/num_eos_tokens": 284, "lr": 1.5681856266894612e-07, "episode": 18320, "epoch": 0.9252525252525252, "step": 1145}, {"eps": 0, "objective/kl": 31.8308162689209, "objective/entropy": -102.07046508789062, "objective/non_score_reward": -1.5915409326553345, "objective/rlhf_reward": 14.752208709716797, "objective/scores": 16.375, "policy/approxkl_avg": 0.0008943321881815791, "policy/clipfrac_avg": 0.006945772562175989, "loss/policy_avg": -0.194179505109787, "loss/value_avg": 0.8305935859680176, "val/clipfrac_avg": 0.00017361111531499773, "policy/entropy_avg": 0.16139507293701172, "val/ratio": 0.9883906245231628, "val/ratio_var": 7.156646461226046e-05, "val/num_eos_tokens": 780, "lr": 1.5351683469185973e-07, "episode": 18336, "epoch": 0.926060606060606, "step": 1146}, {"eps": 0, "objective/kl": 31.00863265991211, "objective/entropy": -73.28153991699219, "objective/non_score_reward": -1.550431728363037, "objective/rlhf_reward": 13.941755294799805, "objective/scores": 15.5, "policy/approxkl_avg": 0.000826484989374876, "policy/clipfrac_avg": 0.004950974136590958, "loss/policy_avg": -0.14431902766227722, "loss/value_avg": 1.0912277698516846, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15498733520507812, "val/ratio": 0.9889597296714783, "val/ratio_var": 6.787945312680677e-05, "val/num_eos_tokens": 441, "lr": 1.5024969360791564e-07, "episode": 18352, "epoch": 0.9268686868686868, "step": 1147}, {"eps": 0, "objective/kl": 37.2110710144043, "objective/entropy": -56.737247467041016, "objective/non_score_reward": -1.860553503036499, "objective/rlhf_reward": 13.061321258544922, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0009301225654780865, "policy/clipfrac_avg": 0.003762050997465849, "loss/policy_avg": -0.03664656728506088, "loss/value_avg": 0.5489372611045837, "val/clipfrac_avg": 0.010529030114412308, "policy/entropy_avg": 0.16833877563476562, "val/ratio": 0.9884514808654785, "val/ratio_var": 3.90233144571539e-05, "val/num_eos_tokens": 126, "lr": 1.4701716273304524e-07, "episode": 18368, "epoch": 0.9276767676767677, "step": 1148}, {"eps": 0, "objective/kl": 41.92686462402344, "objective/entropy": -37.70539855957031, "objective/non_score_reward": -2.0963432788848877, "objective/rlhf_reward": 14.622406959533691, "objective/scores": 16.75, "policy/approxkl_avg": 0.0010944223031401634, "policy/clipfrac_avg": 0.004749140702188015, "loss/policy_avg": -0.13550132513046265, "loss/value_avg": 0.310035765171051, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21954727172851562, "val/ratio": 0.9851934313774109, "val/ratio_var": 6.719871453242376e-05, "val/num_eos_tokens": 114, "lr": 1.4381926513618139e-07, "episode": 18384, "epoch": 0.9284848484848485, "step": 1149}, {"eps": 0, "objective/kl": 38.189300537109375, "objective/entropy": -55.67281723022461, "objective/non_score_reward": -1.9094650745391846, "objective/rlhf_reward": 14.028034210205078, "objective/scores": 15.9375, "policy/approxkl_avg": 0.001011414104141295, "policy/clipfrac_avg": 0.005307096056640148, "loss/policy_avg": -0.06519678980112076, "loss/value_avg": 0.4952123761177063, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20238876342773438, "val/ratio": 0.9863063097000122, "val/ratio_var": 0.00010381833271821961, "val/num_eos_tokens": 103, "lr": 1.4065602363909747e-07, "episode": 18400, "epoch": 0.9292929292929293, "step": 1150}, {"eps": 0, "objective/kl": 39.29536056518555, "objective/entropy": -64.18814086914062, "objective/non_score_reward": -1.9647679328918457, "objective/rlhf_reward": 13.746169090270996, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0009069567313417792, "policy/clipfrac_avg": 0.00267383293248713, "loss/policy_avg": 0.025855571031570435, "loss/value_avg": 0.7742699980735779, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19156980514526367, "val/ratio": 0.9885040521621704, "val/ratio_var": 7.799283775966614e-05, "val/num_eos_tokens": 21, "lr": 1.375274608162447e-07, "episode": 18416, "epoch": 0.9301010101010101, "step": 1151}, {"eps": 0, "objective/kl": 40.618900299072266, "objective/entropy": -59.235267639160156, "objective/non_score_reward": -2.030945062637329, "objective/rlhf_reward": 11.54718017578125, "objective/scores": 13.5625, "policy/approxkl_avg": 0.0008881621761247516, "policy/clipfrac_avg": 0.0032237274572253227, "loss/policy_avg": 0.01431315392255783, "loss/value_avg": 0.6286057829856873, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17874622344970703, "val/ratio": 0.9887373447418213, "val/ratio_var": 5.042510019848123e-05, "val/num_eos_tokens": 353, "lr": 1.3443359899458997e-07, "episode": 18432, "epoch": 0.9309090909090909, "step": 1152}, {"eps": 0, "objective/kl": 46.32222366333008, "objective/entropy": -56.33277130126953, "objective/non_score_reward": -2.3161110877990723, "objective/rlhf_reward": 15.996389389038086, "objective/scores": 18.25, "policy/approxkl_avg": 0.0009288631845265627, "policy/clipfrac_avg": 0.0027455075178295374, "loss/policy_avg": -0.0857156366109848, "loss/value_avg": 0.2431882619857788, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1957530975341797, "val/ratio": 0.9869426488876343, "val/ratio_var": 4.2594107071636245e-05, "val/num_eos_tokens": 26, "lr": 1.3137446025345557e-07, "episode": 18448, "epoch": 0.9317171717171717, "step": 1153}, {"eps": 0, "objective/kl": 33.21459197998047, "objective/entropy": -81.86225891113281, "objective/non_score_reward": -1.6607296466827393, "objective/rlhf_reward": 13.62833309173584, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0008894167840480804, "policy/clipfrac_avg": 0.0032217884436249733, "loss/policy_avg": -0.09154313802719116, "loss/value_avg": 0.5833359360694885, "val/clipfrac_avg": 0.0005091222701594234, "policy/entropy_avg": 0.16341471672058105, "val/ratio": 0.9880057573318481, "val/ratio_var": 7.092337909853086e-05, "val/num_eos_tokens": 503, "lr": 1.283500664243642e-07, "episode": 18464, "epoch": 0.9325252525252525, "step": 1154}, {"eps": 0, "objective/kl": 37.38941192626953, "objective/entropy": -86.53608703613281, "objective/non_score_reward": -1.8694705963134766, "objective/rlhf_reward": 14.591466903686523, "objective/scores": 16.5, "policy/approxkl_avg": 0.0010367620270699263, "policy/clipfrac_avg": 0.007691029459238052, "loss/policy_avg": 0.10637003928422928, "loss/value_avg": 0.5488271713256836, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16927146911621094, "val/ratio": 0.9881411790847778, "val/ratio_var": 8.204267214750871e-05, "val/num_eos_tokens": 399, "lr": 1.253604390908819e-07, "episode": 18480, "epoch": 0.9333333333333333, "step": 1155}, {"eps": 0, "objective/kl": 44.4080810546875, "objective/entropy": -87.75643920898438, "objective/non_score_reward": -2.2204041481018066, "objective/rlhf_reward": 14.404596328735352, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008544453303329647, "policy/clipfrac_avg": 0.0035026988480240107, "loss/policy_avg": -0.024129033088684082, "loss/value_avg": 0.4214504361152649, "val/clipfrac_avg": 9.057971328729764e-05, "policy/entropy_avg": 0.16388320922851562, "val/ratio": 0.9889931082725525, "val/ratio_var": 6.746850704075769e-05, "val/num_eos_tokens": 8, "lr": 1.2240559958846266e-07, "episode": 18496, "epoch": 0.9341414141414142, "step": 1156}, {"eps": 0, "objective/kl": 39.96202850341797, "objective/entropy": -90.10191345214844, "objective/non_score_reward": -1.9981014728546143, "objective/rlhf_reward": 16.14252471923828, "objective/scores": 18.125, "policy/approxkl_avg": 0.0008546580211259425, "policy/clipfrac_avg": 0.005381043069064617, "loss/policy_avg": -0.08957675099372864, "loss/value_avg": 0.3355456590652466, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16065406799316406, "val/ratio": 0.989165186882019, "val/ratio_var": 4.7844900109339505e-05, "val/num_eos_tokens": 3, "lr": 1.1948556900429786e-07, "episode": 18512, "epoch": 0.934949494949495, "step": 1157}, {"eps": 0, "objective/kl": 40.30827331542969, "objective/entropy": -63.458831787109375, "objective/non_score_reward": -2.015413761138916, "objective/rlhf_reward": 13.633024215698242, "objective/scores": 15.625, "policy/approxkl_avg": 0.0008652628748677671, "policy/clipfrac_avg": 0.009679234586656094, "loss/policy_avg": -0.06473330408334732, "loss/value_avg": 0.4225938320159912, "val/clipfrac_avg": 0.000117481205961667, "policy/entropy_avg": 0.1683187484741211, "val/ratio": 0.989500880241394, "val/ratio_var": 3.429090429563075e-05, "val/num_eos_tokens": 301, "lr": 1.1660036817716492e-07, "episode": 18528, "epoch": 0.9357575757575758, "step": 1158}, {"eps": 0, "objective/kl": 36.45595169067383, "objective/entropy": -65.75634765625, "objective/non_score_reward": -1.8227976560592651, "objective/rlhf_reward": 13.138139724731445, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0008474080823361874, "policy/clipfrac_avg": 0.006815841421484947, "loss/policy_avg": 0.19546493887901306, "loss/value_avg": 0.6905479431152344, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16717529296875, "val/ratio": 0.9894546270370483, "val/ratio_var": 7.423999340971932e-05, "val/num_eos_tokens": 124, "lr": 1.1375001769728e-07, "episode": 18544, "epoch": 0.9365656565656566, "step": 1159}, {"eps": 0, "objective/kl": 37.055477142333984, "objective/entropy": -168.40960693359375, "objective/non_score_reward": -1.852773666381836, "objective/rlhf_reward": 14.084726333618164, "objective/scores": 15.9375, "policy/approxkl_avg": 0.0007912840810604393, "policy/clipfrac_avg": 0.0036162808537483215, "loss/policy_avg": 0.10427418351173401, "loss/value_avg": 0.7115825414657593, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1563091278076172, "val/ratio": 0.9901214838027954, "val/ratio_var": 7.212928176159039e-05, "val/num_eos_tokens": 427, "lr": 1.1093453790614883e-07, "episode": 18560, "epoch": 0.9373737373737374, "step": 1160}, {"eps": 0, "objective/kl": 44.201019287109375, "objective/entropy": -70.233154296875, "objective/non_score_reward": -2.2100508213043213, "objective/rlhf_reward": 16.102449417114258, "objective/scores": 18.25, "policy/approxkl_avg": 0.0011408152058720589, "policy/clipfrac_avg": 0.004944411106407642, "loss/policy_avg": -0.12785540521144867, "loss/value_avg": 0.39417845010757446, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19538497924804688, "val/ratio": 0.9865884184837341, "val/ratio_var": 6.0507758462335914e-05, "val/num_eos_tokens": 331, "lr": 1.0815394889642339e-07, "episode": 18576, "epoch": 0.9381818181818182, "step": 1161}, {"eps": 0, "objective/kl": 43.3171501159668, "objective/entropy": -37.648441314697266, "objective/non_score_reward": -2.1658573150634766, "objective/rlhf_reward": 13.099767684936523, "objective/scores": 15.25, "policy/approxkl_avg": 0.0009900203440338373, "policy/clipfrac_avg": 0.003007909283041954, "loss/policy_avg": -0.03844344615936279, "loss/value_avg": 0.655718207359314, "val/clipfrac_avg": 0.0003906250058207661, "policy/entropy_avg": 0.20216941833496094, "val/ratio": 0.9874058961868286, "val/ratio_var": 6.270931771723554e-05, "val/num_eos_tokens": 199, "lr": 1.0540827051175817e-07, "episode": 18592, "epoch": 0.938989898989899, "step": 1162}, {"eps": 0, "objective/kl": 38.83243942260742, "objective/entropy": -83.78005981445312, "objective/non_score_reward": -1.941622018814087, "objective/rlhf_reward": 15.863064765930176, "objective/scores": 17.75, "policy/approxkl_avg": 0.0008689056267030537, "policy/clipfrac_avg": 0.004589558579027653, "loss/policy_avg": 0.05490083992481232, "loss/value_avg": 0.4446883201599121, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1755847930908203, "val/ratio": 0.9878473281860352, "val/ratio_var": 5.755285747000016e-05, "val/num_eos_tokens": 209, "lr": 1.0269752234666642e-07, "episode": 18608, "epoch": 0.9397979797979797, "step": 1163}, {"eps": 0, "objective/kl": 28.182273864746094, "objective/entropy": -106.71968078613281, "objective/non_score_reward": -1.4091136455535889, "objective/rlhf_reward": 13.051824569702148, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0006604374502785504, "policy/clipfrac_avg": 0.006071992218494415, "loss/policy_avg": 0.5652229189872742, "loss/value_avg": 4.454827785491943, "val/clipfrac_avg": 0.0032051282469183207, "policy/entropy_avg": 0.13239336013793945, "val/ratio": 0.9910687208175659, "val/ratio_var": 5.329243867890909e-05, "val/num_eos_tokens": 199, "lr": 1.0002172374638519e-07, "episode": 18624, "epoch": 0.9406060606060606, "step": 1164}, {"eps": 0, "objective/kl": 46.05052947998047, "objective/entropy": -50.39229965209961, "objective/non_score_reward": -2.3025264739990234, "objective/rlhf_reward": 15.775598526000977, "objective/scores": 18.125, "policy/approxkl_avg": 0.0010013967985287309, "policy/clipfrac_avg": 0.004134259186685085, "loss/policy_avg": 0.030554044991731644, "loss/value_avg": 0.3813501298427582, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20552825927734375, "val/ratio": 0.986208975315094, "val/ratio_var": 5.7627206842880696e-05, "val/num_eos_tokens": 147, "lr": 9.738089380673e-08, "episode": 18640, "epoch": 0.9414141414141414, "step": 1165}, {"eps": 0, "objective/kl": 46.88139343261719, "objective/entropy": -48.5864143371582, "objective/non_score_reward": -2.344069719314575, "objective/rlhf_reward": 14.749679565429688, "objective/scores": 17.125, "policy/approxkl_avg": 0.0009588432731106877, "policy/clipfrac_avg": 0.003158081788569689, "loss/policy_avg": 0.014425485394895077, "loss/value_avg": 0.17465442419052124, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19722747802734375, "val/ratio": 0.9872362613677979, "val/ratio_var": 5.964138108538464e-05, "val/num_eos_tokens": 150, "lr": 9.477505137396759e-08, "episode": 18656, "epoch": 0.9422222222222222, "step": 1166}, {"eps": 0, "objective/kl": 38.01190948486328, "objective/entropy": -76.38983154296875, "objective/non_score_reward": -1.9005954265594482, "objective/rlhf_reward": 11.919716835021973, "objective/scores": 13.8125, "policy/approxkl_avg": 0.0007046982645988464, "policy/clipfrac_avg": 0.0028958613984286785, "loss/policy_avg": 0.12084536254405975, "loss/value_avg": 0.5831397771835327, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1511383056640625, "val/ratio": 0.990734875202179, "val/ratio_var": 6.113814743002877e-05, "val/num_eos_tokens": 417, "lr": 9.22042150446728e-08, "episode": 18672, "epoch": 0.943030303030303, "step": 1167}, {"eps": 0, "objective/kl": 44.538299560546875, "objective/entropy": -190.17677307128906, "objective/non_score_reward": -2.226915121078491, "objective/rlhf_reward": 14.882460594177246, "objective/scores": 17.125, "policy/approxkl_avg": 0.0007411800906993449, "policy/clipfrac_avg": 0.005143396090716124, "loss/policy_avg": -0.1721150279045105, "loss/value_avg": 0.29965418577194214, "val/clipfrac_avg": 1.3469827536027879e-05, "policy/entropy_avg": 0.14809703826904297, "val/ratio": 0.990103542804718, "val/ratio_var": 4.799567614099942e-05, "val/num_eos_tokens": 76, "lr": 8.966840316560255e-08, "episode": 18688, "epoch": 0.9438383838383838, "step": 1168}, {"eps": 0, "objective/kl": 43.76121520996094, "objective/entropy": -26.354211807250977, "objective/non_score_reward": -2.1880605220794678, "objective/rlhf_reward": 12.905689239501953, "objective/scores": 15.125, "policy/approxkl_avg": 0.0010139367077499628, "policy/clipfrac_avg": 0.00270761433057487, "loss/policy_avg": -0.04706333950161934, "loss/value_avg": 0.4846585988998413, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19493675231933594, "val/ratio": 0.9869682788848877, "val/ratio_var": 4.458573312149383e-05, "val/num_eos_tokens": 515, "lr": 8.716763383355863e-08, "episode": 18704, "epoch": 0.9446464646464646, "step": 1169}, {"eps": 0, "objective/kl": 33.78557205200195, "objective/entropy": -118.44176483154297, "objective/non_score_reward": -1.6892786026000977, "objective/rlhf_reward": 14.466971397399902, "objective/scores": 16.125, "policy/approxkl_avg": 0.0008025875431485474, "policy/clipfrac_avg": 0.005034630186855793, "loss/policy_avg": -0.11434232443571091, "loss/value_avg": 0.32796037197113037, "val/clipfrac_avg": 0.000240384615608491, "policy/entropy_avg": 0.17252540588378906, "val/ratio": 0.9892752766609192, "val/ratio_var": 8.975582750281319e-05, "val/num_eos_tokens": 534, "lr": 8.470192489526519e-08, "episode": 18720, "epoch": 0.9454545454545454, "step": 1170}, {"eps": 0, "objective/kl": 35.37781524658203, "objective/entropy": -90.45104217529297, "objective/non_score_reward": -1.7688908576965332, "objective/rlhf_reward": 15.840483665466309, "objective/scores": 17.625, "policy/approxkl_avg": 0.0007532388553954661, "policy/clipfrac_avg": 0.00782378576695919, "loss/policy_avg": -0.26330432295799255, "loss/value_avg": 0.43789923191070557, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14811134338378906, "val/ratio": 0.9901248216629028, "val/ratio_var": 4.5044969738228247e-05, "val/num_eos_tokens": 449, "lr": 8.227129394723643e-08, "episode": 18736, "epoch": 0.9462626262626262, "step": 1171}, {"eps": 0, "objective/kl": 45.11640930175781, "objective/entropy": -90.80900573730469, "objective/non_score_reward": -2.2558202743530273, "objective/rlhf_reward": 13.744178771972656, "objective/scores": 16.0, "policy/approxkl_avg": 0.000919578829780221, "policy/clipfrac_avg": 0.005172869190573692, "loss/policy_avg": 0.09692054986953735, "loss/value_avg": 0.2597772777080536, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17873764038085938, "val/ratio": 0.9880226850509644, "val/ratio_var": 5.284639337332919e-05, "val/num_eos_tokens": 400, "lr": 7.987575833565298e-08, "episode": 18752, "epoch": 0.9470707070707071, "step": 1172}, {"eps": 0, "objective/kl": 37.627655029296875, "objective/entropy": -163.6809844970703, "objective/non_score_reward": -1.8813828229904175, "objective/rlhf_reward": 14.59518051147461, "objective/scores": 16.5, "policy/approxkl_avg": 0.0007881260244175792, "policy/clipfrac_avg": 0.0037529754918068647, "loss/policy_avg": 0.09063289314508438, "loss/value_avg": 0.5753982663154602, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15079402923583984, "val/ratio": 0.9891691207885742, "val/ratio_var": 4.976944182999432e-05, "val/num_eos_tokens": 262, "lr": 7.7515335156238e-08, "episode": 18768, "epoch": 0.9478787878787879, "step": 1173}, {"eps": 0, "objective/kl": 38.040130615234375, "objective/entropy": -171.2418670654297, "objective/non_score_reward": -1.9020065069198608, "objective/rlhf_reward": 13.176118850708008, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0006656637415289879, "policy/clipfrac_avg": 0.0033770238514989614, "loss/policy_avg": 0.14138484001159668, "loss/value_avg": 0.9088690280914307, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1490926742553711, "val/ratio": 0.9910387992858887, "val/ratio_var": 5.571794099523686e-05, "val/num_eos_tokens": 584, "lr": 7.519004125413454e-08, "episode": 18784, "epoch": 0.9486868686868687, "step": 1174}, {"eps": 0, "objective/kl": 43.64716339111328, "objective/entropy": -50.938758850097656, "objective/non_score_reward": -2.1823582649230957, "objective/rlhf_reward": 13.802017211914062, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009952617110684514, "policy/clipfrac_avg": 0.0027715780306607485, "loss/policy_avg": -0.09168200194835663, "loss/value_avg": 0.44036585092544556, "val/clipfrac_avg": 0.0004111842135898769, "policy/entropy_avg": 0.17894935607910156, "val/ratio": 0.987088680267334, "val/ratio_var": 6.252353341551498e-05, "val/num_eos_tokens": 169, "lr": 7.289989322378732e-08, "episode": 18800, "epoch": 0.9494949494949495, "step": 1175}, {"eps": 0, "objective/kl": 43.219486236572266, "objective/entropy": -153.3197479248047, "objective/non_score_reward": -2.1609742641448975, "objective/rlhf_reward": 12.741369247436523, "objective/scores": 14.875, "policy/approxkl_avg": 0.0009114840067923069, "policy/clipfrac_avg": 0.004199516959488392, "loss/policy_avg": -0.17822237312793732, "loss/value_avg": 0.4382031559944153, "val/clipfrac_avg": 3.814697265625e-05, "policy/entropy_avg": 0.18891334533691406, "val/ratio": 0.9868748784065247, "val/ratio_var": 7.712288061156869e-05, "val/num_eos_tokens": 23, "lr": 7.064490740882057e-08, "episode": 18816, "epoch": 0.9503030303030303, "step": 1176}, {"eps": 0, "objective/kl": 46.933937072753906, "objective/entropy": -62.70560073852539, "objective/non_score_reward": -2.3466968536376953, "objective/rlhf_reward": 15.286115646362305, "objective/scores": 17.625, "policy/approxkl_avg": 0.0009256636258214712, "policy/clipfrac_avg": 0.005345791578292847, "loss/policy_avg": -0.29926538467407227, "loss/value_avg": 0.24289564788341522, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19078445434570312, "val/ratio": 0.9878906607627869, "val/ratio_var": 5.454176425701007e-05, "val/num_eos_tokens": 267, "lr": 6.842509990192536e-08, "episode": 18832, "epoch": 0.9511111111111111, "step": 1177}, {"eps": 0, "objective/kl": 38.56993103027344, "objective/entropy": -102.2330322265625, "objective/non_score_reward": -1.9284967184066772, "objective/rlhf_reward": 13.352752685546875, "objective/scores": 15.25, "policy/approxkl_avg": 0.0008128330809995532, "policy/clipfrac_avg": 0.004654319025576115, "loss/policy_avg": -0.07624176144599915, "loss/value_avg": 1.0276535749435425, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16016387939453125, "val/ratio": 0.9896093010902405, "val/ratio_var": 5.181640881346539e-05, "val/num_eos_tokens": 33, "lr": 6.624048654474247e-08, "episode": 18848, "epoch": 0.9519191919191919, "step": 1178}, {"eps": 0, "objective/kl": 39.02704620361328, "objective/entropy": -59.69445037841797, "objective/non_score_reward": -1.9513523578643799, "objective/rlhf_reward": 13.173646926879883, "objective/scores": 15.125, "policy/approxkl_avg": 0.001039088936522603, "policy/clipfrac_avg": 0.003585438709706068, "loss/policy_avg": 0.035814136266708374, "loss/value_avg": 0.5764297842979431, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19232654571533203, "val/ratio": 0.9868942499160767, "val/ratio_var": 6.958071753615513e-05, "val/num_eos_tokens": 192, "lr": 6.409108292774912e-08, "episode": 18864, "epoch": 0.9527272727272728, "step": 1179}, {"eps": 0, "objective/kl": 47.32844543457031, "objective/entropy": -40.500762939453125, "objective/non_score_reward": -2.366422176361084, "objective/rlhf_reward": 13.852327346801758, "objective/scores": 16.25, "policy/approxkl_avg": 0.0010082677472382784, "policy/clipfrac_avg": 0.005784696899354458, "loss/policy_avg": 0.24797323346138, "loss/value_avg": 0.2845846712589264, "val/clipfrac_avg": 0.00013241525448393077, "policy/entropy_avg": 0.19600677490234375, "val/ratio": 0.9869298934936523, "val/ratio_var": 4.6343844587681815e-05, "val/num_eos_tokens": 504, "lr": 6.197690439015025e-08, "episode": 18880, "epoch": 0.9535353535353536, "step": 1180}, {"eps": 0, "objective/kl": 38.190425872802734, "objective/entropy": -87.35359191894531, "objective/non_score_reward": -1.9095213413238525, "objective/rlhf_reward": 12.574853897094727, "objective/scores": 14.5, "policy/approxkl_avg": 0.000693869951646775, "policy/clipfrac_avg": 0.0022745702881366014, "loss/policy_avg": -0.011056587100028992, "loss/value_avg": 0.21195590496063232, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14525604248046875, "val/ratio": 0.9902079105377197, "val/ratio_var": 5.0829687097575516e-05, "val/num_eos_tokens": 672, "lr": 5.989796601976516e-08, "episode": 18896, "epoch": 0.9543434343434344, "step": 1181}, {"eps": 0, "objective/kl": 36.28763961791992, "objective/entropy": -131.93319702148438, "objective/non_score_reward": -1.8143819570541382, "objective/rlhf_reward": 12.099679946899414, "objective/scores": 13.9375, "policy/approxkl_avg": 0.0006996375159360468, "policy/clipfrac_avg": 0.0033093849197030067, "loss/policy_avg": 0.5624556541442871, "loss/value_avg": 0.936620831489563, "val/clipfrac_avg": 0.0014880952658131719, "policy/entropy_avg": 0.1407628059387207, "val/ratio": 0.9909154176712036, "val/ratio_var": 5.887184670427814e-05, "val/num_eos_tokens": 240, "lr": 5.785428265292381e-08, "episode": 18912, "epoch": 0.9551515151515152, "step": 1182}, {"eps": 0, "objective/kl": 39.66671371459961, "objective/entropy": -117.6522216796875, "objective/non_score_reward": -1.9833356142044067, "objective/rlhf_reward": 13.297914505004883, "objective/scores": 15.25, "policy/approxkl_avg": 0.0007932945154607296, "policy/clipfrac_avg": 0.0029290905222296715, "loss/policy_avg": -0.19112437963485718, "loss/value_avg": 0.6247477531433105, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16626358032226562, "val/ratio": 0.9893531799316406, "val/ratio_var": 5.722875357605517e-05, "val/num_eos_tokens": 41, "lr": 5.584586887435739e-08, "episode": 18928, "epoch": 0.955959595959596, "step": 1183}, {"eps": 0, "objective/kl": 38.85816955566406, "objective/entropy": -59.193870544433594, "objective/non_score_reward": -1.942908525466919, "objective/rlhf_reward": 12.28365421295166, "objective/scores": 14.25, "policy/approxkl_avg": 0.0009058800060302019, "policy/clipfrac_avg": 0.004390705842524767, "loss/policy_avg": -0.018872268497943878, "loss/value_avg": 0.6814364194869995, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17277050018310547, "val/ratio": 0.9888806939125061, "val/ratio_var": 5.870430322829634e-05, "val/num_eos_tokens": 391, "lr": 5.387273901709733e-08, "episode": 18944, "epoch": 0.9567676767676768, "step": 1184}, {"eps": 0, "objective/kl": 35.59428405761719, "objective/entropy": -72.95974731445312, "objective/non_score_reward": -1.7797143459320068, "objective/rlhf_reward": 13.673410415649414, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0008201617165468633, "policy/clipfrac_avg": 0.004145351238548756, "loss/policy_avg": -0.12193582952022552, "loss/value_avg": 0.4072027802467346, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17438912391662598, "val/ratio": 0.9888736605644226, "val/ratio_var": 6.133368151495233e-05, "val/num_eos_tokens": 60, "lr": 5.1934907162370374e-08, "episode": 18960, "epoch": 0.9575757575757575, "step": 1185}, {"eps": 0, "objective/kl": 41.09333038330078, "objective/entropy": -97.86048889160156, "objective/non_score_reward": -2.054666519165039, "objective/rlhf_reward": 13.710959434509277, "objective/scores": 15.75, "policy/approxkl_avg": 0.000780026544816792, "policy/clipfrac_avg": 0.007402490358799696, "loss/policy_avg": 0.3409832715988159, "loss/value_avg": 2.1178839206695557, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19989013671875, "val/ratio": 0.9890344738960266, "val/ratio_var": 7.444098446285352e-05, "val/num_eos_tokens": 76, "lr": 5.0032387139501425e-08, "episode": 18976, "epoch": 0.9583838383838383, "step": 1186}, {"eps": 0, "objective/kl": 36.85295486450195, "objective/entropy": -31.46831512451172, "objective/non_score_reward": -1.842647671699524, "objective/rlhf_reward": 14.922977447509766, "objective/scores": 16.75, "policy/approxkl_avg": 0.001208727015182376, "policy/clipfrac_avg": 0.004555144812911749, "loss/policy_avg": -0.16587065160274506, "loss/value_avg": 0.5858635902404785, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.20450401306152344, "val/ratio": 0.9874365329742432, "val/ratio_var": 5.309836706146598e-05, "val/num_eos_tokens": 113, "lr": 4.8165192525809754e-08, "episode": 18992, "epoch": 0.9591919191919192, "step": 1187}, {"eps": 0, "objective/kl": 32.94627380371094, "objective/entropy": -105.82904815673828, "objective/non_score_reward": -1.6473137140274048, "objective/rlhf_reward": 15.290185928344727, "objective/scores": 17.0, "policy/approxkl_avg": 0.0006628286791965365, "policy/clipfrac_avg": 0.004865474533289671, "loss/policy_avg": 0.06852813065052032, "loss/value_avg": 0.32328563928604126, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.12561321258544922, "val/ratio": 0.9917237758636475, "val/ratio_var": 3.8588947063544765e-05, "val/num_eos_tokens": 202, "lr": 4.63333366465174e-08, "episode": 19008, "epoch": 0.96, "step": 1188}, {"eps": 0, "objective/kl": 45.81948471069336, "objective/entropy": -33.98373031616211, "objective/non_score_reward": -2.2909743785858154, "objective/rlhf_reward": 13.005900382995605, "objective/scores": 15.3125, "policy/approxkl_avg": 0.0010197311639785767, "policy/clipfrac_avg": 0.003383245086297393, "loss/policy_avg": 0.020303204655647278, "loss/value_avg": 0.4330519437789917, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.21395492553710938, "val/ratio": 0.985183835029602, "val/ratio_var": 5.527482062461786e-05, "val/num_eos_tokens": 86, "lr": 4.4536832574650355e-08, "episode": 19024, "epoch": 0.9608080808080808, "step": 1189}, {"eps": 0, "objective/kl": 38.252220153808594, "objective/entropy": -152.57052612304688, "objective/non_score_reward": -1.9126112461090088, "objective/rlhf_reward": 14.89988899230957, "objective/scores": 16.75, "policy/approxkl_avg": 0.0007638135575689375, "policy/clipfrac_avg": 0.0035591432824730873, "loss/policy_avg": -0.6133148074150085, "loss/value_avg": 0.3876040279865265, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16634774208068848, "val/ratio": 0.9892970323562622, "val/ratio_var": 6.732134352205321e-05, "val/num_eos_tokens": 392, "lr": 4.2775693130948094e-08, "episode": 19040, "epoch": 0.9616161616161616, "step": 1190}, {"eps": 0, "objective/kl": 46.120994567871094, "objective/entropy": -81.75244140625, "objective/non_score_reward": -2.3060498237609863, "objective/rlhf_reward": 13.537699699401855, "objective/scores": 15.875, "policy/approxkl_avg": 0.0009323159465566278, "policy/clipfrac_avg": 0.003273975569754839, "loss/policy_avg": -0.16719648241996765, "loss/value_avg": 0.4944162368774414, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19600319862365723, "val/ratio": 0.9876390695571899, "val/ratio_var": 4.03721714974381e-05, "val/num_eos_tokens": 291, "lr": 4.104993088376974e-08, "episode": 19056, "epoch": 0.9624242424242424, "step": 1191}, {"eps": 0, "objective/kl": 38.58598327636719, "objective/entropy": -58.236507415771484, "objective/non_score_reward": -1.9292993545532227, "objective/rlhf_reward": 13.148825645446777, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0010641806293278933, "policy/clipfrac_avg": 0.004020705819129944, "loss/policy_avg": 0.08641587942838669, "loss/value_avg": 0.6021695137023926, "val/clipfrac_avg": 0.00021150591783225536, "policy/entropy_avg": 0.19188880920410156, "val/ratio": 0.9875043630599976, "val/ratio_var": 7.918935443740338e-05, "val/num_eos_tokens": 665, "lr": 3.935955814900527e-08, "episode": 19072, "epoch": 0.9632323232323232, "step": 1192}, {"eps": 0, "objective/kl": 36.086280822753906, "objective/entropy": -126.37530517578125, "objective/non_score_reward": -1.8043140172958374, "objective/rlhf_reward": 13.906623840332031, "objective/scores": 15.6875, "policy/approxkl_avg": 0.0008227983489632607, "policy/clipfrac_avg": 0.004573613405227661, "loss/policy_avg": 0.09802606701850891, "loss/value_avg": 1.0540528297424316, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15534448623657227, "val/ratio": 0.9889199733734131, "val/ratio_var": 9.045511978911236e-05, "val/num_eos_tokens": 325, "lr": 3.77045869899878e-08, "episode": 19088, "epoch": 0.964040404040404, "step": 1193}, {"eps": 0, "objective/kl": 32.869319915771484, "objective/entropy": -35.11339569091797, "objective/non_score_reward": -1.6434661149978638, "objective/rlhf_reward": 14.551846504211426, "objective/scores": 16.25, "policy/approxkl_avg": 0.002314617158845067, "policy/clipfrac_avg": 0.0031052969861775637, "loss/policy_avg": -0.17688646912574768, "loss/value_avg": 0.4370724558830261, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17627334594726562, "val/ratio": 0.989388108253479, "val/ratio_var": 5.49936194147449e-05, "val/num_eos_tokens": 185, "lr": 3.608502921740753e-08, "episode": 19104, "epoch": 0.9648484848484848, "step": 1194}, {"eps": 0, "objective/kl": 40.820533752441406, "objective/entropy": -92.06233978271484, "objective/non_score_reward": -2.0410265922546387, "objective/rlhf_reward": 14.513660430908203, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009747116710059345, "policy/clipfrac_avg": 0.005887703038752079, "loss/policy_avg": -0.17504875361919403, "loss/value_avg": 0.204596608877182, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17911338806152344, "val/ratio": 0.9880841970443726, "val/ratio_var": 4.2761934309965e-05, "val/num_eos_tokens": 294, "lr": 3.450089638922738e-08, "episode": 19120, "epoch": 0.9656565656565657, "step": 1195}, {"eps": 0, "objective/kl": 45.62751007080078, "objective/entropy": -24.155014038085938, "objective/non_score_reward": -2.2813756465911865, "objective/rlhf_reward": 14.195186614990234, "objective/scores": 16.5, "policy/approxkl_avg": 0.0010361537570133805, "policy/clipfrac_avg": 0.006471775472164154, "loss/policy_avg": -0.12304341048002243, "loss/value_avg": 0.32093346118927, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19795608520507812, "val/ratio": 0.9868389368057251, "val/ratio_var": 5.5822583817644045e-05, "val/num_eos_tokens": 209, "lr": 3.2952199810599714e-08, "episode": 19136, "epoch": 0.9664646464646465, "step": 1196}, {"eps": 0, "objective/kl": 31.232709884643555, "objective/entropy": -111.23834228515625, "objective/non_score_reward": -1.5616354942321777, "objective/rlhf_reward": 13.35242748260498, "objective/scores": 14.9375, "policy/approxkl_avg": 0.0007176597137004137, "policy/clipfrac_avg": 0.004162969067692757, "loss/policy_avg": 0.3443071246147156, "loss/value_avg": 1.5372172594070435, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2422410249710083, "val/ratio": 0.9907166957855225, "val/ratio_var": 7.462361827492714e-05, "val/num_eos_tokens": 524, "lr": 3.143895053378698e-08, "episode": 19152, "epoch": 0.9672727272727273, "step": 1197}, {"eps": 0, "objective/kl": 32.114402770996094, "objective/entropy": -101.44007110595703, "objective/non_score_reward": -1.6057199239730835, "objective/rlhf_reward": 14.378654479980469, "objective/scores": 16.0, "policy/approxkl_avg": 0.000817152438685298, "policy/clipfrac_avg": 0.004995419643819332, "loss/policy_avg": -0.13895125687122345, "loss/value_avg": 0.5284525752067566, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15488815307617188, "val/ratio": 0.9890648126602173, "val/ratio_var": 7.766807539155707e-05, "val/num_eos_tokens": 443, "lr": 2.9961159358082305e-08, "episode": 19168, "epoch": 0.9680808080808081, "step": 1198}, {"eps": 0, "objective/kl": 37.39568328857422, "objective/entropy": -71.81754302978516, "objective/non_score_reward": -1.8697842359542847, "objective/rlhf_reward": 13.520840644836426, "objective/scores": 15.375, "policy/approxkl_avg": 0.0008745577652007341, "policy/clipfrac_avg": 0.002502672839909792, "loss/policy_avg": -0.2223341315984726, "loss/value_avg": 0.6924841403961182, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18534183502197266, "val/ratio": 0.9876028299331665, "val/ratio_var": 5.801619045087136e-05, "val/num_eos_tokens": 599, "lr": 2.8518836829732332e-08, "episode": 19184, "epoch": 0.9688888888888889, "step": 1199}, {"eps": 0, "objective/kl": 31.942792892456055, "objective/entropy": -74.75447082519531, "objective/non_score_reward": -1.597139596939087, "objective/rlhf_reward": 13.043485641479492, "objective/scores": 14.625, "policy/approxkl_avg": 0.0007055438472889364, "policy/clipfrac_avg": 0.0035820305347442627, "loss/policy_avg": 0.15578943490982056, "loss/value_avg": 0.9613189101219177, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15447449684143066, "val/ratio": 0.9913029670715332, "val/ratio_var": 5.023244739277288e-05, "val/num_eos_tokens": 539, "lr": 2.7111993241860646e-08, "episode": 19200, "epoch": 0.9696969696969697, "step": 1200}, {"eps": 0, "objective/kl": 37.90913391113281, "objective/entropy": -63.07292175292969, "objective/non_score_reward": -1.8954565525054932, "objective/rlhf_reward": 14.112356185913086, "objective/scores": 16.0, "policy/approxkl_avg": 0.0008319137268699706, "policy/clipfrac_avg": 0.0048570772632956505, "loss/policy_avg": -0.1969611942768097, "loss/value_avg": 0.46793222427368164, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15787792205810547, "val/ratio": 0.9898661375045776, "val/ratio_var": 3.5913133615395054e-05, "val/num_eos_tokens": 176, "lr": 2.5740638634397243e-08, "episode": 19216, "epoch": 0.9705050505050505, "step": 1201}, {"eps": 0, "objective/kl": 42.020835876464844, "objective/entropy": -85.2213134765625, "objective/non_score_reward": -2.101041793823242, "objective/rlhf_reward": 14.586458206176758, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009388537146151066, "policy/clipfrac_avg": 0.004320736508816481, "loss/policy_avg": -0.0762750431895256, "loss/value_avg": 0.7102093696594238, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18203043937683105, "val/ratio": 0.9878751635551453, "val/ratio_var": 8.284598879981786e-05, "val/num_eos_tokens": 108, "lr": 2.440478279400471e-08, "episode": 19232, "epoch": 0.9713131313131314, "step": 1202}, {"eps": 0, "objective/kl": 33.10966491699219, "objective/entropy": -50.760467529296875, "objective/non_score_reward": -1.6554831266403198, "objective/rlhf_reward": 14.21170425415039, "objective/scores": 15.875, "policy/approxkl_avg": 0.0010394756682217121, "policy/clipfrac_avg": 0.005734467413276434, "loss/policy_avg": 0.023966491222381592, "loss/value_avg": 0.36540549993515015, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18365859985351562, "val/ratio": 0.9874485731124878, "val/ratio_var": 6.813264917582273e-05, "val/num_eos_tokens": 456, "lr": 2.3104435254008852e-08, "episode": 19248, "epoch": 0.9721212121212122, "step": 1203}, {"eps": 0, "objective/kl": 34.464874267578125, "objective/entropy": -41.6898307800293, "objective/non_score_reward": -1.7232437133789062, "objective/rlhf_reward": 14.409568786621094, "objective/scores": 16.125, "policy/approxkl_avg": 0.0008781763026490808, "policy/clipfrac_avg": 0.0046472325921058655, "loss/policy_avg": 0.11596876382827759, "loss/value_avg": 0.8759022951126099, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18979644775390625, "val/ratio": 0.988035261631012, "val/ratio_var": 7.465987437171862e-05, "val/num_eos_tokens": 8, "lr": 2.1839605294330935e-08, "episode": 19264, "epoch": 0.972929292929293, "step": 1204}, {"eps": 0, "objective/kl": 38.246883392333984, "objective/entropy": -64.19001770019531, "objective/non_score_reward": -1.912344217300415, "objective/rlhf_reward": 12.517343521118164, "objective/scores": 14.4375, "policy/approxkl_avg": 0.0008579955901950598, "policy/clipfrac_avg": 0.002436618786305189, "loss/policy_avg": 0.033114898949861526, "loss/value_avg": 0.37955981492996216, "val/clipfrac_avg": 0.006591796875, "policy/entropy_avg": 0.18629741668701172, "val/ratio": 0.9880963563919067, "val/ratio_var": 3.743668276001699e-05, "val/num_eos_tokens": 791, "lr": 2.0610301941420552e-08, "episode": 19280, "epoch": 0.9737373737373738, "step": 1205}, {"eps": 0, "objective/kl": 38.28486633300781, "objective/entropy": -47.87005615234375, "objective/non_score_reward": -1.9142433404922485, "objective/rlhf_reward": 15.296693801879883, "objective/scores": 17.25, "policy/approxkl_avg": 0.0010683179134503007, "policy/clipfrac_avg": 0.004513275809586048, "loss/policy_avg": -0.37829723954200745, "loss/value_avg": 0.47165149450302124, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2166946828365326, "val/ratio": 0.9851523041725159, "val/ratio_var": 8.785770478425547e-05, "val/num_eos_tokens": 24, "lr": 1.9416533968193428e-08, "episode": 19296, "epoch": 0.9745454545454545, "step": 1206}, {"eps": 0, "objective/kl": 45.99113464355469, "objective/entropy": -79.00741577148438, "objective/non_score_reward": -2.2995567321777344, "objective/rlhf_reward": 13.708255767822266, "objective/scores": 16.0, "policy/approxkl_avg": 0.0009223736124113202, "policy/clipfrac_avg": 0.0040555731393396854, "loss/policy_avg": -0.12085376679897308, "loss/value_avg": 0.5372148156166077, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17890167236328125, "val/ratio": 0.9878267049789429, "val/ratio_var": 6.0516715166158974e-05, "val/num_eos_tokens": 75, "lr": 1.8258309893965375e-08, "episode": 19312, "epoch": 0.9753535353535353, "step": 1207}, {"eps": 0, "objective/kl": 42.04967498779297, "objective/entropy": -66.35103607177734, "objective/non_score_reward": -2.1024839878082275, "objective/rlhf_reward": 14.780328750610352, "objective/scores": 16.875, "policy/approxkl_avg": 0.0009762570261955261, "policy/clipfrac_avg": 0.004208541475236416, "loss/policy_avg": -0.08923263102769852, "loss/value_avg": 0.5102915167808533, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19926166534423828, "val/ratio": 0.987288236618042, "val/ratio_var": 6.778948591090739e-05, "val/num_eos_tokens": 505, "lr": 1.713563798439455e-08, "episode": 19328, "epoch": 0.9761616161616161, "step": 1208}, {"eps": 0, "objective/kl": 42.122718811035156, "objective/entropy": -112.43741607666016, "objective/non_score_reward": -2.106135845184326, "objective/rlhf_reward": 11.198551177978516, "objective/scores": 13.3125, "policy/approxkl_avg": 0.0009249926079064608, "policy/clipfrac_avg": 0.004424647428095341, "loss/policy_avg": 0.10433784127235413, "loss/value_avg": 0.8428288698196411, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19216203689575195, "val/ratio": 0.9879051446914673, "val/ratio_var": 6.320722604868934e-05, "val/num_eos_tokens": 27, "lr": 1.6048526251421502e-08, "episode": 19344, "epoch": 0.9769696969696969, "step": 1209}, {"eps": 0, "objective/kl": 34.459083557128906, "objective/entropy": -77.9478530883789, "objective/non_score_reward": -1.722954273223877, "objective/rlhf_reward": 14.105171203613281, "objective/scores": 15.8125, "policy/approxkl_avg": 0.001020944444462657, "policy/clipfrac_avg": 0.004141935613006353, "loss/policy_avg": 0.16060814261436462, "loss/value_avg": 0.33783212304115295, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18428802490234375, "val/ratio": 0.9876895546913147, "val/ratio_var": 0.00010520015348447487, "val/num_eos_tokens": 65, "lr": 1.4996982453211463e-08, "episode": 19360, "epoch": 0.9777777777777777, "step": 1210}, {"eps": 0, "objective/kl": 35.14643096923828, "objective/entropy": -46.33203125, "objective/non_score_reward": -1.7573217153549194, "objective/rlhf_reward": 13.656740188598633, "objective/scores": 15.4375, "policy/approxkl_avg": 0.0009761269902810454, "policy/clipfrac_avg": 0.003752158023416996, "loss/policy_avg": -0.06701843440532684, "loss/value_avg": 0.3668726682662964, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19565200805664062, "val/ratio": 0.9875230193138123, "val/ratio_var": 6.482725439127535e-05, "val/num_eos_tokens": 46, "lr": 1.3981014094099354e-08, "episode": 19376, "epoch": 0.9785858585858586, "step": 1211}, {"eps": 0, "objective/kl": 28.53516387939453, "objective/entropy": -69.5890884399414, "objective/non_score_reward": -1.426758050918579, "objective/rlhf_reward": 14.3076171875, "objective/scores": 15.75, "policy/approxkl_avg": 0.0006685786065645516, "policy/clipfrac_avg": 0.0029422249644994736, "loss/policy_avg": -0.2390209585428238, "loss/value_avg": 0.7244433760643005, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16259002685546875, "val/ratio": 0.9912441968917847, "val/ratio_var": 4.31347252742853e-05, "val/num_eos_tokens": 143, "lr": 1.3000628424535978e-08, "episode": 19392, "epoch": 0.9793939393939394, "step": 1212}, {"eps": 0, "objective/kl": 41.615177154541016, "objective/entropy": -82.7415542602539, "objective/non_score_reward": -2.080759048461914, "objective/rlhf_reward": 14.606740951538086, "objective/scores": 16.75, "policy/approxkl_avg": 0.0009461919544264674, "policy/clipfrac_avg": 0.006980390287935734, "loss/policy_avg": -0.11228986084461212, "loss/value_avg": 0.5388200283050537, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18620681762695312, "val/ratio": 0.987679123878479, "val/ratio_var": 6.926806236151606e-05, "val/num_eos_tokens": 134, "lr": 1.2055832441036918e-08, "episode": 19408, "epoch": 0.9802020202020202, "step": 1213}, {"eps": 0, "objective/kl": 38.947208404541016, "objective/entropy": -104.67575073242188, "objective/non_score_reward": -1.9473603963851929, "objective/rlhf_reward": 14.27138900756836, "objective/scores": 16.25, "policy/approxkl_avg": 0.0007925564423203468, "policy/clipfrac_avg": 0.0025324940215796232, "loss/policy_avg": -0.20061983168125153, "loss/value_avg": 0.2869341969490051, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15836048126220703, "val/ratio": 0.9894617795944214, "val/ratio_var": 4.7174267820082605e-05, "val/num_eos_tokens": 256, "lr": 1.1146632886132603e-08, "episode": 19424, "epoch": 0.981010101010101, "step": 1214}, {"eps": 0, "objective/kl": 41.90412521362305, "objective/entropy": -101.33221435546875, "objective/non_score_reward": -2.0952064990997314, "objective/rlhf_reward": 14.529792785644531, "objective/scores": 16.625, "policy/approxkl_avg": 0.0008722887723706663, "policy/clipfrac_avg": 0.0025138547644019127, "loss/policy_avg": -0.12940247356891632, "loss/value_avg": 0.5062823295593262, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17874455451965332, "val/ratio": 0.9877287149429321, "val/ratio_var": 6.869715434731916e-05, "val/num_eos_tokens": 248, "lr": 1.0273036248318325e-08, "episode": 19440, "epoch": 0.9818181818181818, "step": 1215}, {"eps": 0, "objective/kl": 35.63471984863281, "objective/entropy": -192.84432983398438, "objective/non_score_reward": -1.781736135482788, "objective/rlhf_reward": 11.92138957977295, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0009429135243408382, "policy/clipfrac_avg": 0.0026213740929961205, "loss/policy_avg": -0.39164668321609497, "loss/value_avg": 1.550127387046814, "val/clipfrac_avg": 7.62939453125e-06, "policy/entropy_avg": 0.17586326599121094, "val/ratio": 0.9886311888694763, "val/ratio_var": 7.116834603948519e-05, "val/num_eos_tokens": 61, "lr": 9.435048762010957e-09, "episode": 19456, "epoch": 0.9826262626262626, "step": 1216}, {"eps": 0, "objective/kl": 39.298187255859375, "objective/entropy": -42.98954391479492, "objective/non_score_reward": -1.964909315109253, "objective/rlhf_reward": 14.472591400146484, "objective/scores": 16.5, "policy/approxkl_avg": 0.0009566507651470602, "policy/clipfrac_avg": 0.004461781587451696, "loss/policy_avg": 0.08832584321498871, "loss/value_avg": 0.4303143620491028, "val/clipfrac_avg": 0.001953125, "policy/entropy_avg": 0.18021583557128906, "val/ratio": 0.9864565134048462, "val/ratio_var": 3.8086534914327785e-05, "val/num_eos_tokens": 243, "lr": 8.632676407502316e-09, "episode": 19472, "epoch": 0.9834343434343434, "step": 1217}, {"eps": 0, "objective/kl": 37.92512512207031, "objective/entropy": -107.35826110839844, "objective/non_score_reward": -1.896256446838379, "objective/rlhf_reward": 13.080305099487305, "objective/scores": 15.0, "policy/approxkl_avg": 0.0009007140179164708, "policy/clipfrac_avg": 0.0045885005965828896, "loss/policy_avg": 0.21013334393501282, "loss/value_avg": 0.4151275157928467, "val/clipfrac_avg": 0.0007440476329065859, "policy/entropy_avg": 0.18993210792541504, "val/ratio": 0.9882104396820068, "val/ratio_var": 7.511641160817817e-05, "val/num_eos_tokens": 164, "lr": 7.865924910916977e-09, "episode": 19488, "epoch": 0.9842424242424243, "step": 1218}, {"eps": 0, "objective/kl": 42.538116455078125, "objective/entropy": -103.62552642822266, "objective/non_score_reward": -2.126905918121338, "objective/rlhf_reward": 13.982468605041504, "objective/scores": 16.125, "policy/approxkl_avg": 0.0008123687002807856, "policy/clipfrac_avg": 0.004256902262568474, "loss/policy_avg": 0.1631181836128235, "loss/value_avg": 0.3172687888145447, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16496849060058594, "val/ratio": 0.9897924661636353, "val/ratio_var": 3.71673668269068e-05, "val/num_eos_tokens": 67, "lr": 7.13479974417175e-09, "episode": 19504, "epoch": 0.9850505050505051, "step": 1219}, {"eps": 0, "objective/kl": 44.76129150390625, "objective/entropy": -120.69778442382812, "objective/non_score_reward": -2.2380645275115967, "objective/rlhf_reward": 14.80099868774414, "objective/scores": 17.0, "policy/approxkl_avg": 0.0009488088544458151, "policy/clipfrac_avg": 0.0036685834638774395, "loss/policy_avg": -0.18715566396713257, "loss/value_avg": 0.5615487098693848, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.17595529556274414, "val/ratio": 0.9875380992889404, "val/ratio_var": 7.448292308254167e-05, "val/num_eos_tokens": 949, "lr": 6.439306124936262e-09, "episode": 19520, "epoch": 0.9858585858585859, "step": 1220}, {"eps": 0, "objective/kl": 42.00151062011719, "objective/entropy": -120.10693359375, "objective/non_score_reward": -2.1000754833221436, "objective/rlhf_reward": 14.212424278259277, "objective/scores": 16.25, "policy/approxkl_avg": 0.0008749752887524664, "policy/clipfrac_avg": 0.004988180007785559, "loss/policy_avg": -0.4247804880142212, "loss/value_avg": 0.9726248979568481, "val/clipfrac_avg": 0.00023134966613724828, "policy/entropy_avg": 0.1530447006225586, "val/ratio": 0.9896517992019653, "val/ratio_var": 5.084315125714056e-05, "val/num_eos_tokens": 39, "lr": 5.779449016595773e-09, "episode": 19536, "epoch": 0.9866666666666667, "step": 1221}, {"eps": 0, "objective/kl": 36.87531280517578, "objective/entropy": -129.04684448242188, "objective/non_score_reward": -1.8437657356262207, "objective/rlhf_reward": 13.230453491210938, "objective/scores": 15.0625, "policy/approxkl_avg": 0.0007634559297002852, "policy/clipfrac_avg": 0.0055130645632743835, "loss/policy_avg": 0.17928610742092133, "loss/value_avg": 0.8346552848815918, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14986801147460938, "val/ratio": 0.9903774857521057, "val/ratio_var": 5.780787250841968e-05, "val/num_eos_tokens": 563, "lr": 5.155233128216197e-09, "episode": 19552, "epoch": 0.9874747474747475, "step": 1222}, {"eps": 0, "objective/kl": 39.414207458496094, "objective/entropy": -49.08431625366211, "objective/non_score_reward": -1.9707105159759521, "objective/rlhf_reward": 13.771476745605469, "objective/scores": 15.75, "policy/approxkl_avg": 0.0010028551332652569, "policy/clipfrac_avg": 0.005500209983438253, "loss/policy_avg": -0.028081759810447693, "loss/value_avg": 0.3895527124404907, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.2012653350830078, "val/ratio": 0.987224280834198, "val/ratio_var": 4.987093052477576e-05, "val/num_eos_tokens": 38, "lr": 4.56666291450858e-09, "episode": 19568, "epoch": 0.9882828282828283, "step": 1223}, {"eps": 0, "objective/kl": 37.389583587646484, "objective/entropy": -40.7100830078125, "objective/non_score_reward": -1.8694791793823242, "objective/rlhf_reward": 13.505520820617676, "objective/scores": 15.375, "policy/approxkl_avg": 0.001194866606965661, "policy/clipfrac_avg": 0.005203597247600555, "loss/policy_avg": -0.014931255020201206, "loss/value_avg": 0.4837211072444916, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.22698020935058594, "val/ratio": 0.9855931401252747, "val/ratio_var": 7.236239616759121e-05, "val/num_eos_tokens": 375, "lr": 4.0137425758018935e-09, "episode": 19584, "epoch": 0.9890909090909091, "step": 1224}, {"eps": 0, "objective/kl": 40.458518981933594, "objective/entropy": -75.71792602539062, "objective/non_score_reward": -2.022926092147827, "objective/rlhf_reward": 14.328636169433594, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009783877758309245, "policy/clipfrac_avg": 0.002967006526887417, "loss/policy_avg": 0.05002763122320175, "loss/value_avg": 0.33766478300094604, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.194549560546875, "val/ratio": 0.9863731265068054, "val/ratio_var": 4.958164936397225e-05, "val/num_eos_tokens": 90, "lr": 3.496476058006959e-09, "episode": 19600, "epoch": 0.98989898989899, "step": 1225}, {"eps": 0, "objective/kl": 45.668174743652344, "objective/entropy": -77.89369201660156, "objective/non_score_reward": -2.2834086418151855, "objective/rlhf_reward": 14.654090881347656, "objective/scores": 17.0, "policy/approxkl_avg": 0.000869191309902817, "policy/clipfrac_avg": 0.0027626235969364643, "loss/policy_avg": -0.09883704781532288, "loss/value_avg": 0.40289461612701416, "val/clipfrac_avg": 0.0002861721732188016, "policy/entropy_avg": 0.18253517150878906, "val/ratio": 0.9884761571884155, "val/ratio_var": 5.125232564751059e-05, "val/num_eos_tokens": 48, "lr": 3.014867052593684e-09, "episode": 19616, "epoch": 0.9907070707070708, "step": 1226}, {"eps": 0, "objective/kl": 41.73491668701172, "objective/entropy": -133.7528839111328, "objective/non_score_reward": -2.0867457389831543, "objective/rlhf_reward": 13.960128784179688, "objective/scores": 16.0, "policy/approxkl_avg": 0.0007204288267530501, "policy/clipfrac_avg": 0.00297390203922987, "loss/policy_avg": 0.02128005027770996, "loss/value_avg": 0.34802740812301636, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14430570602416992, "val/ratio": 0.9903041124343872, "val/ratio_var": 6.99873999110423e-05, "val/num_eos_tokens": 316, "lr": 2.568918996560532e-09, "episode": 19632, "epoch": 0.9915151515151515, "step": 1227}, {"eps": 0, "objective/kl": 42.037235260009766, "objective/entropy": -44.44089126586914, "objective/non_score_reward": -2.1018619537353516, "objective/rlhf_reward": 14.179388046264648, "objective/scores": 16.25, "policy/approxkl_avg": 0.0009157764725387096, "policy/clipfrac_avg": 0.003242512932047248, "loss/policy_avg": -0.30528533458709717, "loss/value_avg": 0.7530245780944824, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19690704345703125, "val/ratio": 0.9876857995986938, "val/ratio_var": 5.984380186419003e-05, "val/num_eos_tokens": 429, "lr": 2.1586350724134285e-09, "episode": 19648, "epoch": 0.9923232323232323, "step": 1228}, {"eps": 0, "objective/kl": 43.7442741394043, "objective/entropy": -50.87998962402344, "objective/non_score_reward": -2.187213897705078, "objective/rlhf_reward": 15.812786102294922, "objective/scores": 18.0, "policy/approxkl_avg": 0.0009651206783019006, "policy/clipfrac_avg": 0.003591865533962846, "loss/policy_avg": 0.07529310882091522, "loss/value_avg": 0.36114734411239624, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19472503662109375, "val/ratio": 0.9876270294189453, "val/ratio_var": 4.56350389868021e-05, "val/num_eos_tokens": 331, "lr": 1.7840182081396707e-09, "episode": 19664, "epoch": 0.9931313131313131, "step": 1229}, {"eps": 0, "objective/kl": 38.04791259765625, "objective/entropy": -69.65153503417969, "objective/non_score_reward": -1.9023958444595337, "objective/rlhf_reward": 13.425728797912598, "objective/scores": 15.3125, "policy/approxkl_avg": 0.000832535675726831, "policy/clipfrac_avg": 0.003121623769402504, "loss/policy_avg": -0.08426038920879364, "loss/value_avg": 0.3430386185646057, "val/clipfrac_avg": 0.0010775862028822303, "policy/entropy_avg": 0.17328572273254395, "val/ratio": 0.9891126155853271, "val/ratio_var": 5.81457388761919e-05, "val/num_eos_tokens": 327, "lr": 1.4450710771907184e-09, "episode": 19680, "epoch": 0.9939393939393939, "step": 1230}, {"eps": 0, "objective/kl": 36.740516662597656, "objective/entropy": -27.681867599487305, "objective/non_score_reward": -1.8370258808135986, "objective/rlhf_reward": 13.741098403930664, "objective/scores": 15.5625, "policy/approxkl_avg": 0.0009004836902022362, "policy/clipfrac_avg": 0.005135446786880493, "loss/policy_avg": 0.002345144748687744, "loss/value_avg": 1.1214191913604736, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18401336669921875, "val/ratio": 0.9884154796600342, "val/ratio_var": 7.396236469503492e-05, "val/num_eos_tokens": 137, "lr": 1.1417960984605459e-09, "episode": 19696, "epoch": 0.9947474747474747, "step": 1231}, {"eps": 0, "objective/kl": 45.00796890258789, "objective/entropy": -45.30944061279297, "objective/non_score_reward": -2.250398635864258, "objective/rlhf_reward": 15.554288864135742, "objective/scores": 17.75, "policy/approxkl_avg": 0.0009604179067537189, "policy/clipfrac_avg": 0.004513026215136051, "loss/policy_avg": -0.16311845183372498, "loss/value_avg": 0.45753657817840576, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.18356657028198242, "val/ratio": 0.9889092445373535, "val/ratio_var": 4.4623157009482384e-05, "val/num_eos_tokens": 4, "lr": 8.741954362678773e-10, "episode": 19712, "epoch": 0.9955555555555555, "step": 1232}, {"eps": 0, "objective/kl": 37.37506103515625, "objective/entropy": -19.211952209472656, "objective/non_score_reward": -1.8687530755996704, "objective/rlhf_reward": 12.959371566772461, "objective/scores": 14.8125, "policy/approxkl_avg": 0.000893141608685255, "policy/clipfrac_avg": 0.003839832032099366, "loss/policy_avg": -0.2625172734260559, "loss/value_avg": 0.8098385334014893, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.1701374053955078, "val/ratio": 0.9891498684883118, "val/ratio_var": 7.125152478693053e-05, "val/num_eos_tokens": 12, "lr": 6.422710003439747e-10, "episode": 19728, "epoch": 0.9963636363636363, "step": 1233}, {"eps": 0, "objective/kl": 46.30189895629883, "objective/entropy": -53.28070831298828, "objective/non_score_reward": -2.3150949478149414, "objective/rlhf_reward": 14.263029098510742, "objective/scores": 16.625, "policy/approxkl_avg": 0.001009408151730895, "policy/clipfrac_avg": 0.006422281265258789, "loss/policy_avg": 0.10235726833343506, "loss/value_avg": 0.41452595591545105, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.19713115692138672, "val/ratio": 0.9869934320449829, "val/ratio_var": 9.03628533706069e-05, "val/num_eos_tokens": 349, "lr": 4.4602444581542945e-10, "episode": 19744, "epoch": 0.9971717171717172, "step": 1234}, {"eps": 0, "objective/kl": 37.48680877685547, "objective/entropy": -68.4776611328125, "objective/non_score_reward": -1.874340534210205, "objective/rlhf_reward": 14.492847442626953, "objective/scores": 16.375, "policy/approxkl_avg": 0.0009436773834750056, "policy/clipfrac_avg": 0.005421591456979513, "loss/policy_avg": -0.13045719265937805, "loss/value_avg": 0.8233026266098022, "val/clipfrac_avg": 4.183400233159773e-05, "policy/entropy_avg": 0.1775188446044922, "val/ratio": 0.988511860370636, "val/ratio_var": 5.036559741711244e-05, "val/num_eos_tokens": 52, "lr": 2.854571731947253e-10, "episode": 19760, "epoch": 0.997979797979798, "step": 1235}, {"eps": 0, "objective/kl": 44.79389190673828, "objective/entropy": -155.7568359375, "objective/non_score_reward": -2.239694595336914, "objective/rlhf_reward": 14.119680404663086, "objective/scores": 16.375, "policy/approxkl_avg": 0.0007429021643474698, "policy/clipfrac_avg": 0.0029837298206984997, "loss/policy_avg": -0.21915020048618317, "loss/value_avg": 0.6580561995506287, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.14424514770507812, "val/ratio": 0.9901254773139954, "val/ratio_var": 6.483635661425069e-05, "val/num_eos_tokens": 405, "lr": 1.6057032836913623e-10, "episode": 19776, "epoch": 0.9987878787878788, "step": 1236}, {"eps": 0, "objective/kl": 38.987640380859375, "objective/entropy": -55.40583801269531, "objective/non_score_reward": -1.9493823051452637, "objective/rlhf_reward": 12.292805671691895, "objective/scores": 14.25, "policy/approxkl_avg": 0.0008245522622019053, "policy/clipfrac_avg": 0.002606890397146344, "loss/policy_avg": 0.19540390372276306, "loss/value_avg": 1.0495953559875488, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.16810131072998047, "val/ratio": 0.9887499809265137, "val/ratio_var": 4.211545819998719e-05, "val/num_eos_tokens": 134, "lr": 7.136480259239964e-11, "episode": 19792, "epoch": 0.9995959595959596, "step": 1237}, {"eps": 0, "objective/kl": 39.11707305908203, "objective/entropy": -112.83091735839844, "objective/non_score_reward": -1.9558537006378174, "objective/rlhf_reward": 11.758989334106445, "objective/scores": 13.6875, "policy/approxkl_avg": 0.0008375335019081831, "policy/clipfrac_avg": 0.004157694987952709, "loss/policy_avg": 0.025396723300218582, "loss/value_avg": 0.5493187308311462, "val/clipfrac_avg": 0.0, "policy/entropy_avg": 0.15214157104492188, "val/ratio": 0.9900014400482178, "val/ratio_var": 6.577932072104886e-05, "val/num_eos_tokens": 299, "lr": 1.784123247916547e-11, "episode": 19808, "epoch": 1.0004040404040404, "step": 1238}], "memory": 0} diff --git a/ppo_dir_v2/v0-20260524-184726/runs/events.out.tfevents.1779619672.TENCENT64.site.230518.0 b/ppo_dir_v2/v0-20260524-184726/runs/events.out.tfevents.1779619672.TENCENT64.site.230518.0 new file mode 100644 index 0000000000000000000000000000000000000000..3a37edc0dff7dd20671df737081a66d6a3acbdd1 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/runs/events.out.tfevents.1779619672.TENCENT64.site.230518.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7145be1896f051dbad7d422f77d7db6406512ab616b8f7dbd33cd55060b5866 +size 1317437 diff --git a/ppo_dir_v2/v0-20260524-184726/val_dataset.jsonl b/ppo_dir_v2/v0-20260524-184726/val_dataset.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cc98b3b0c384d78b01783185b6c062c1bab83918 --- /dev/null +++ b/ppo_dir_v2/v0-20260524-184726/val_dataset.jsonl @@ -0,0 +1,200 @@ +{"messages": [{"role": "user", "content": "Read the following article and write a one sentence summary.\n(Insert Article)"}, {"role": "assistant", "content": "Without an article inserted, it is not possible for me to provide a summary. Please insert an article so that I can read it and provide a summary."}]} +{"messages": [{"role": "user", "content": "How can we reduce air pollution?"}, {"role": "assistant", "content": "There are several ways to reduce air pollution, including:\n\n1. Reduce energy consumption: By conserving energy, we reduce the amount of pollution emitted from power plants. Some ways to achieve this include: using energy-efficient appliances and lighting, insulating homes, and shutting off lights and electronics when not in use.\n\n2. Use public transportation or carpool: Cars and trucks are major sources of pollution. By using public transportation, carpooling or biking, you can reduce the number of cars on the road, which in turn reduces emissions.\n\n3. Be responsible with waste: When waste is burned or decomposed, they can release harmful gases into the air. Dispose of hazardous waste correctly and recycle materials like plastics, paper, and glass when possible.\n\n4. Support clean energy sources: Renewable energy sources like solar and wind power generate little or no pollution. Support clean energy companies and projects, and consider installing renewable energy systems in your home.\n\n5. Drive efficiently: If you need to drive, make sure your vehicle is well maintained and drive efficiently by avoiding rapid acceleration and braking, and reducing idling.\n\n6. Be mindful of product choices: Some products and practices release more pollution than others. Choose products with eco-friendly packaging and try to minimize your use of aerosol sprays, which can release harmful chemicals into the air."}]} +{"messages": [{"role": "user", "content": "Rewrite the following sentence by replacing all the words with synonyms to make the sentence clearer.\nQuite a few people alleged that the criminal had committed a serious offence"}, {"role": "assistant", "content": "Several individuals claimed that the offender had perpetrated a grave transgression."}]} +{"messages": [{"role": "user", "content": "Rearrange this sentence and identify the kinds of words used\nBeneath the waves sea, the sponges swayed"}, {"role": "assistant", "content": "The sponges swayed beneath the sea waves.\n\nTypes of word used:\n- Article: \"The\"\n- Noun: \"Sponges\", \"sea\", \"waves\"\n- Verb: \"swayed\"\n- Preposition: \"Beneath\"\n- Adjective: None\n- Pronoun: None\n- Adverb: None\n- Conjunction: None."}]} +{"messages": [{"role": "user", "content": "Generate a recipe for vegan brownies without eggs."}, {"role": "assistant", "content": "Here is a simple recipe for vegan brownies without eggs:\n\nIngredients:\n- 1.5 cups of flour \n- 3 tablespoons of cocoa powder\n- 1/2 teaspoon of baking powder \n- 1/2 teaspoon of salt\n- 1 cup of sugar \n- 1/2 cup of vegetable oil \n- 1/2 cup of water \n- 1 teaspoon of vanilla extract\n- 1/2 cup of vegan chocolate chips\n\nInstructions:\n\n1. Preheat your oven to 350 degrees Fahrenheit (180 degrees Celsius).\n2. In a large mixing bowl, sift together the flour, cocoa powder, baking powder, and salt.\n3. In a separate mixing bowl, whisk together the sugar, vegetable oil, water, and vanilla extract.\n4. Combine the wet and dry ingredients and mix until the batter is smooth.\n5. Fold in the vegan chocolate chips. \n6. Pour the batter into an 8×8 inch greased baking pan.\n7. Bake for 25 to 30 minutes or until the edges of the brownies pull away from the sides of the pan and a toothpick inserted into the center comes out clean.\n8. Let the brownies cool completely before cutting into squares and serving.\n\nEnjoy your delicious egg-free vegan brownies!"}]} +{"messages": [{"role": "user", "content": "Create a job description for an experienced accountant"}, {"role": "assistant", "content": "Job Title: Experienced Accountant\n\nJob Summary:\nWe are seeking an experienced accountant to join our dynamic team. The ideal candidate will be responsible for ensuring the accuracy and financial integrity of the company’s accounting records, and will play a key role in financial planning, budgeting, analysis and reporting.\n\nResponsibilities:\n- Manage the company’s financial accounts, ensuring compliance with local and international accounting standards.\n- Prepare and present accurate and timely financial statements, reports and forecasts to management and other stakeholders.\n- Participate in budget planning and monitoring activities, and provide relevant insights for decision making.\n- Maintain and update accounting systems, and implement process improvements where necessary.\n- Oversee tax compliance, prepare and file tax returns in a timely manner.\n- Ensure the implementation of effective internal controls to minimize financial risks.\n- Liaise with external auditors as required, and respond to audit queries in a timely and professional manner.\n- Participate in ad-hoc projects and analyses as required.\n\nRequirements:\n- Bachelor's degree in Accounting, Finance or related field.\n- Minimum of 5 years’ experience in a similar role.\n- Excellent knowledge of accounting standards and principles.\n- Advanced proficiency in the use of accounting software.\n- Strong analytical, problem solving and decision-making skills.\n- Excellent verbal and written communication skills.\n- CPA or other relevant certification is highly desirable.\n\nWe offer a competitive salary package, opportunities for professional development, and a collaborative work environment. If you are a self-starter with a passion for excellence, we encourage you to apply."}]} +{"messages": [{"role": "user", "content": "Fill in the blank with the most appropriate verb according to the context.\nWe wrote about the match, the players' strategies and their incredible _______."}, {"role": "assistant", "content": "performance."}]} +{"messages": [{"role": "user", "content": "Give an example of how art can be used to bring about social change."}, {"role": "assistant", "content": "Art has long been a powerful tool for bringing about social change. One example of this can be seen in the work of artist and activist Keith Haring, who used his bold and colorful visual art to raise awareness and speak out about social and political issues such as AIDS, racism, and war. Haring's art was not only visually striking and thought-provoking, but it also helped to bring attention to important issues and inspire people to take action. Through his art, Haring was able to engage communities, start conversations, and inspire action, showing that art can be a powerful tool for enacting social change."}]} +{"messages": [{"role": "user", "content": "Name five attractions near the Grand Canyon."}, {"role": "assistant", "content": "1. Havasu Falls: Located in the Havasupai Indian Reservation, Havasu Falls is an oasis in the desert south of the Grand Canyon. This is a beautiful waterfall with turquoise blue water and a must-see for nature enthusiasts.\n\n2. Antelope Canyon: Often referred to as 'corkscrew canyon,' Antelope Canyon is a slot canyon located east of the Grand Canyon. With stunning light beams and narrow, colorful walls, it's a famous and breathtaking attraction.\n\n3. Horseshoe Bend: This is a horseshoe-shaped meander of the Colorado River around a rock formation. Located near Page, Arizona, Horseshoe Bend offers stunning views over the river and the desert landscape.\n\n4. Desert View Watchtower: Located on the South Rim of the Grand Canyon, Desert View Watchtower gives visitors a spectacular 360-degree view of the Grand Canyon and the Painted Desert. This 70-feet tall stone tower was designed by Mary Colter and offers breathtaking vistas.\n\n5. Grand Canyon Skywalk: Located at the West Rim of the Grand Canyon, the Skywalk is a horseshoe-shaped, glass-bottom bridge that extends 70 feet over the edge of the Canyon, allowing visitors to walk on air and see straight down to the bottom of the Canyon over 4,000 feet below."}]} +{"messages": [{"role": "user", "content": "Access the third element of the list.\n[0, 1, 6, 3, 4]"}, {"role": "assistant", "content": "To access the third element of the list [0, 1, 6, 3, 4], you can use indexing by referring to the index number of the desired element, enclosed in square brackets. Keep in mind that indexing starts from 0 in most programming languages. Here is the code to access the third element of the list:\n\n```python\nmy_list = [0, 1, 6, 3, 4]\nthird_element = my_list[2]\nprint(third_element)\n```\n\nThe result of this code would be `6`, as the element with the index number `2` is the third element in the list."}]} +{"messages": [{"role": "user", "content": "What is the most common type of data visualization used?"}, {"role": "assistant", "content": "The most common types of data visualization used are charts and graphs, such as bar charts, pie charts, line graphs, and scatter plots. These types of visualizations are widely used because they are effective in presenting data in a clear and easy-to-understand manner."}]} +{"messages": [{"role": "user", "content": "Write a creative story or poem based on the following prompt.\nA wizard and a dragon met in a strange world."}, {"role": "assistant", "content": "In a land far away,\nPast the mountains of gray,\nA wizard and a dragon met,\nIn a strange world, an unusual threat.\n\nThe wizard, wise and old,\nCarried a staff of gold,\nA deep wisdom in his eyes,\nAnd a knowledge of the skies.\n\nThe dragon, fierce and bold,\nWith scales of red and gold,\nA fiery breath and piercing roar,\nA creature of myth and lore.\n\nThey stood facing each other,\nIn a world like no other,\nA place of magic and wonder,\nWhere the sky was torn asunder.\n\nThe dragon growled and snarled,\nAnd its wings fiercely unfurled,\nBut the wizard stood his ground,\nAnd raised his staff without a sound.\n\nA beam of light shot forth,\nFrom the tip of the staff of worth,\nIt struck the dragon’s heart,\nAnd the creature fell apart.\n\nBut instead of death and demise,\nThe dragon began to rise,\nIts form changing, taking shape,\nInto a creature of beauty and grace.\n\nThe wizard had not killed,\nBut had instead fulfilled,\nA spell of transformation,\nFor the dragon’s own salvation.\n\nAnd so, the wizard and the dragon,\nNow friends, not foes, in tandem,\nWalked within that strange world,\nTheir friendship forever unfurled."}]} +{"messages": [{"role": "user", "content": "Convert the following to a yes or no question:\nIs the color red?"}, {"role": "assistant", "content": "Is the color red? Yes or no?"}]} +{"messages": [{"role": "user", "content": "Find the most crucial points in the following paragraph.\nThe coronavirus pandemic has affected every part of the world, hitting some countries harder than others. For example, the United States has had over 6.9 million cases and more than 200,000 deaths. Governments worldwide are responding in various ways, from ramping up testing to instituting lockdowns to implementing plans for reopening."}, {"role": "assistant", "content": "- The coronavirus pandemic has affected every part of the world\n- Some countries have been hit harder than others\n- The United States has had over 6.9 million cases and more than 200,000 deaths\n- Governments worldwide are responding in various ways including increasing testing, instituting lockdowns and implementing reopening plans."}]} +{"messages": [{"role": "user", "content": "Suggest a plausible narrative for a short story about a woman discovering her inner-strength."}, {"role": "assistant", "content": "Once there was a woman named Mary who was always under the impression that her worth is defined by the opinions of others. Consequently, she lived her life meeting up with the expectations and validation of others, until life threw her a curveball when her husband, whom she loved very much, suddenly passed away in a tragic accident.\n\nConsumed with grief, she had to endure the pain of losing a loved one, but she faced another challenge, of raising her two children singlehandedly. This new reality made her realize how little faith she had in her capabilities.\n\nMary struggled to make ends meet, taking up multiple part-time jobs while also taking care of her children's needs. Every night she would cry herself to sleep, feeling helpless and weak, until one day, when she stumbled upon her old college journal. As she read through the pages, she remembered her dreams and aspirations that she had let go of because she never believed in herself.\n\nThat moment was a turning point in her life. She decided to take life by its horns and started to work towards her long lost dream of starting her own business. It wasn't an easy road, and it was filled with obstacles, but she persevered, and slowly but surely, her business started to take off.\n\nThrough this journey, Mary discovered her inner strength that she never knew existed. She realized that her worth was not defined by others but by her belief in herself. And with this new realization, she emerged as a strong and confident woman, who not only ran a successful business but also raised two beautiful children, who looked up to her as their role model."}]} +{"messages": [{"role": "user", "content": "Imagine that you are a reporter. Write a news report of 50 words or less based on the following input.\nFlorida Governor Ron DeSantis introduced a bill proposing to make all public areas of the state smoke-free."}, {"role": "assistant", "content": "Florida Governor Ron DeSantis recently introduced a bill to ban smoking in all public spaces across the state. This proposed measure aims to promote healthier living among Florida residents, and reduce second-hand smoke exposure in public areas."}]} +{"messages": [{"role": "user", "content": "Assign a logical operator to each of the following terms: true, false, and."}, {"role": "assistant", "content": "The terms True and False are boolean values, not operators. The term 'and' is a logical operator that takes two boolean values and returns True only if both values are True, otherwise, it returns False."}]} +{"messages": [{"role": "user", "content": "What are the different ways of treating depression?"}, {"role": "assistant", "content": "Depression is a common and serious medical illness that affects how a person feels, thinks, and behaves. Fortunately, it is also treatable. There are several ways of treating depression, which may include one or a combination of the following:\n\n1. Psychotherapy: This includes therapies such as cognitive-behavioral therapy (CBT), interpersonal therapy (IPT), and problem-solving therapy. These therapies can help people with depression change negative thinking patterns and improve their relationships and coping skills.\n\n2. Antidepressant medication: These medications work by balancing certain chemicals in the brain that affect mood and emotions. They can help reduce symptoms of depression and improve the quality of life for many people.\n\n3. Exercise and healthy lifestyle: Physical activity can boost mood, reduce stress, and improve sleep. Eating a healthy diet, getting enough sleep, and avoiding alcohol and drugs can also help improve symptoms of depression.\n\n4. Light therapy: This therapy involves using a bright light box for a specific amount of time each day. It can help regulate mood and improve symptoms of depression, especially for those who experience seasonal depression.\n\n5. Brain stimulation therapies: This includes methods such as electroconvulsive therapy (ECT), transcranial magnetic stimulation (TMS), and vagus nerve stimulation (VNS). These therapies can be effective for those who do not respond well to medication or psychotherapy.\n\n6. Support groups and self-help: Joining a support group or practicing relaxation techniques and mindfulness can help some people cope with depression and feel less alone.\n\nIt's important to note that every person is different and may respond differently to various treatments. Working with a qualified healthcare provider to find the right treatment approach is crucial. It's also important to remember that just like any serious illness, depression should be addressed immediately and it is really important that people seek help."}]} +{"messages": [{"role": "user", "content": "Identify the most populous country in the world."}, {"role": "assistant", "content": "According to the most recent data, the most populous country in the world is China, with an estimated population of 1.398 billion people as of 2021."}]} +{"messages": [{"role": "user", "content": "Analyze the following sentence and identify the type of speech - indirect speech or direct speech\nShe said she was feeling tired."}, {"role": "assistant", "content": "The sentence is an example of indirect speech."}]} +{"messages": [{"role": "user", "content": "List five virtues that the given person possess.\nNicole Kidman"}, {"role": "assistant", "content": "1. Kindness: Nicole Kidman is known for being a very kind and warm-hearted person, often going out of her way to help others.\n\n2. Compassion: She's deeply empathetic and cares about the well-being of those around her, often supporting various charities and humanitarian causes.\n\n3. Resilience: Nicole has faced several personal and professional challenges in her life but has always bounced back and remained strong.\n\n4. Determination: In her pursuit of her career, Nicole has shown a tremendous amount of drive and determination, which has helped her succeed in the industry.\n\n5. Professionalism: Nicole is widely respected for her work ethic and her ability to consistently deliver high-quality performances."}]} +{"messages": [{"role": "user", "content": "Determine the function of this sentence in the given text.\nAlso, studies have documented increased patient satisfaction."}, {"role": "assistant", "content": "The function of this sentence in the given text appears to be to provide evidence or support for an argument or point being made. It presents information suggesting that there are studies which show a positive outcome (increased patient satisfaction) likely related to the topic being discussed in the text."}]} +{"messages": [{"role": "user", "content": "Describe the differences between anaerobic and aerobic exercise."}, {"role": "assistant", "content": "Aerobic and anaerobic exercise are two forms of physical activity that work the body in different ways. The main difference between the two types of exercise is the presence or absence of oxygen.\n\nAerobic exercise, also known as cardio, is a type of physical activity that requires oxygen to produce energy. This type of exercise typically involves rhythmic movements that increase the heart rate and breathing rate, allowing the body to take in more oxygen to fuel the muscles. Common examples of aerobic exercise include running, cycling, swimming, and dancing.\n\nAnaerobic exercise, on the other hand, is a type of physical activity that does not require oxygen to produce energy. Instead, the body relies on other energy sources, such as stored carbohydrates, to fuel intense, short bursts of activity. This type of exercise is typically more intense than aerobic exercise and involves activities such as weightlifting, sprinting, and jumping.\n\nIn summary, aerobic exercise uses oxygen to produce energy and is typically performed at a moderate intensity for longer periods of time, while anaerobic exercise relies on other energy sources to fuel short bursts of intense activity. Both types of exercise offer numerous health benefits and can be incorporated into a well-rounded fitness routine."}]} +{"messages": [{"role": "user", "content": "Look up the given word and provide its definition.\nApathy"}, {"role": "assistant", "content": "Apathy refers to a lack of interest or concern, particularly in regards to matters of general importance. It is the state of being emotionally uninvolved, indifferent, or unmoved by something or someone. In essence, it's a feeling of dispassion or numbness, often accompanied by an absence of motivation, enthusiasm, or excitement."}]} +{"messages": [{"role": "user", "content": "Create a reward system for employees that shows appreciation but also encourages productivity."}, {"role": "assistant", "content": "One way to create a reward system for employees that shows appreciation and encourages productivity is through a points-based program. Here's how it could work:\n\n1. Employees earn points for hitting specific targets or achieving certain goals. For example, they could earn points for meeting project deadlines, exceeding monthly sales targets, or for coming up with innovative ideas that benefit the company.\n\n2. The points accumulated by employees can then be redeemed for various incentives. These incentives could be in the form of tangible rewards such as gift cards, electronics, or additional vacation days. Or they could be experiences, such as a team-building event, a company-sponsored outing, or a paid training or conference.\n\n3. The reward system should be transparent and easy to understand. Employees should know exactly how many points they need to earn to redeem specific rewards, and there should be a clear pathway to earning more points.\n\n4. To encourage productivity, the rewards offered should be attractive and desirable to employees. The company could survey its employees to find out what kinds of rewards would motivate them the most.\n\n5. Lastly, it's important to celebrate employees' achievements and recognize their efforts. The company could have a monthly or quarterly award ceremony where employees are recognized for earning the most points and redeeming the most rewards.\n\nOverall, a well-designed reward system can show appreciation for employees while also encouraging productivity by providing tangible incentives for meeting and exceeding goals."}]} +{"messages": [{"role": "user", "content": "Predict how this statement would be received by the audience.\nWe should begin to reduce our use of single-use plastics in our daily lives."}, {"role": "assistant", "content": "The reception of this statement would likely vary among different audience members. Some individuals might be receptive and supportive of reducing single-use plastics in their daily lives, understanding the importance and long-term impact of such actions on the environment. Others might feel indifferent or resistant to the statement, preferring the convenience of single-use plastics, and may not fully understand the long-term environmental consequences. Overall, the statement may be seen as a call to action by some, and as a suggestion that may be disregarded by others."}]} +{"messages": [{"role": "user", "content": "List 5 ways to reduce the number of car accidents."}, {"role": "assistant", "content": "1. Distracted driving prevention - Implementing laws banning the use of hand-held devices, including cell phones, texting, and navigating while driving can reduce distractions and improve road safety.\n\n2. Strict enforcement of drunk driving laws - Stricter legal consequences for driving under the influence of alcohol or drugs, coupled with increased awareness campaigns, can help deter drunk driving, which is a common cause for accidents.\n\n3. Regular vehicle maintenance - Ensuring vehicles are regularly maintained and checked can prevent accidents due to technical failures. This includes regular checks on brakes, tires, alignment, and lighting systems.\n\n4. Better road design - Improving road infrastructure and design can also help reduce accidents. This includes better signage, improved visibility, efficient traffic flow, and reducing road hazards.\n\n5. Speed limits - Setting and enforcing appropriate speed limits for different roads and weather conditions can help prevent accidents. Speeding significantly increases the risk and severity of accidents, so enforcing speed limits can help save lives."}]} +{"messages": [{"role": "user", "content": "Find a movie quote that will inspire me to take a risk."}, {"role": "assistant", "content": "Here's a quote from the movie Dead Poets Society (1989) by the character John Keating played by Robin Williams that might inspire you: \"Carpe diem. Seize the day, boys. Make your lives extraordinary.\""}]} +{"messages": [{"role": "user", "content": "Create a custom greeting for a visitor.\nThe visitor's name is John."}, {"role": "assistant", "content": "Hello John, welcome! I'm so glad you're here. How can I assist you today?"}]} +{"messages": [{"role": "user", "content": "Read the article and identify the central idea.\nHere is the article. \n\nIndia is a country of varied cultures, languages, and religions. Each of these elements co-exist peacefully here, giving India its unique and rich identity. People from different backgrounds live in harmony and form strong bonds of friendship which are the foundation of our great nation."}, {"role": "assistant", "content": "The central idea of the article is about the diversity of cultures, languages, and religions in India and how these elements coexist peacefully, forming strong bonds and contributing to the unique and rich identity of the nation."}]} +{"messages": [{"role": "user", "content": "Write a code snippet for generating a random number."}, {"role": "assistant", "content": "In Python, the `random` module provides a suite of functions to perform stochastic operations. Here's an example of generating a random number between two given values, 1 and 10, using the `randint()` function from `random`:\n\n```python\nimport random\n\n# generate a random integer between 1 and 10\nnumber = random.randint(1, 10)\n\n# print the generated number\nprint(f\"Random number between 1 and 10: {number}\")\n```\n\nThe `randint()` function returns a random integer from the specified range. In this example, it will return a value between 1 and 10, inclusive."}]} +{"messages": [{"role": "user", "content": "Create a recipe for a healthy breakfast that can be cooked in 10 minutes or less."}, {"role": "assistant", "content": "Here is a delicious, healthy breakfast recipe that can be cooked in 10 minutes or less:\n\n**Avocado Toast with Egg and Spinach**\n\nIngredients:\n- 2 slices of whole-wheat bread\n- 1 avocado\n- 1 tablespoon of lemon juice\n- Salt and pepper\n- 2 large eggs\n- A handful of spinach leaves\n\nInstructions:\n1. Begin by toasting two slices of whole-wheat bread.\n2. While the bread is toasting, mash the avocado in a small bowl and mix it with 1 tablespoon of lemon juice, salt, and pepper to taste.\n3. Heat a non-stick pan over medium heat, then fry 2 large eggs to your liking.\n4. While the eggs cook, spread the mashed avocado mixture onto the toasted bread slices.\n5. Gently lay a handful of spinach leaves over the avocado, then place the cooked eggs over the top.\n6. Sprinkle with more pepper and enjoy your healthy and filling breakfast!\n\nThis recipe is quick to make, and it's loaded with healthy fats, protein, fiber, and vitamins. Enjoy!"}]} +{"messages": [{"role": "user", "content": "Give an example of a bias that could exist in an AI algorithm."}, {"role": "assistant", "content": "One example of a bias that could exist in an AI algorithm is gender bias. This can occur when an algorithm is trained on historical data that reflects existing societal gender biases, leading the algorithm to make predictions or decisions based on those biases. For instance, if an AI system used for hiring is trained on data from a company with a history of male-dominated leadership, the algorithm may \"learn\" to favor male candidates over equally or more qualified female candidates due to the patterns it has detected in the training data. This can perpetuate gender inequality and result in unfair treatment of female candidates."}]} +{"messages": [{"role": "user", "content": "Given a sentence, generate the next logical sentence.\nI received a good grade on my test."}, {"role": "assistant", "content": "This improved my overall average in the class."}]} +{"messages": [{"role": "user", "content": "Compare two airlines and identify better services.\nNorthwest Airlines\nSouthwest Airlines"}, {"role": "assistant", "content": "When comparing Northwest Airlines to Southwest Airlines, it is important to consider the criteria for evaluating \"better\" services. If we define \"better\" as having a more positive reputation for customer satisfaction, on-time performance, and cost-effectiveness, then Southwest Airlines stands out. According to a J.D. Power's 2021 North America Airline Satisfaction Study, Southwest Airlines ranks highest among low-cost carriers in overall customer satisfaction. The carrier consistently receives high marks in areas such as customer service, handling of bags, and boarding/deplaning processes. Additionally, Southwest Airlines has a reputation for providing competitive prices and allowing customers to change flights without fees or penalties. \n\nIt is important to note that Northwest Airlines is no longer operational after its merger with Delta Air Lines in 2010. Delta Airlines ranks near the top of most customer satisfaction surveys and often offers a more comprehensive network of routes and flights than Southwest Airlines. Ultimately, the \"better\" airline will depend on the priorities and preferences of each individual traveler."}]} +{"messages": [{"role": "user", "content": "Create a new logo for the given firm.\nFerris Consulting"}, {"role": "assistant", "content": "I am an AI text-based assistant and do not have the ability to create a physical logo. However, I suggest incorporating elements such as a Ferris wheel or a stylized letter \"F\" to represent the name \"Ferris\". You can use a sophisticated and modern font to create an elegant yet contemporary look. Using a combination of colors such as dark blue and gold can give the logo a professional and distinguished appearance. You could also experiment with geometric shapes or abstract designs to create a unique and memorable logo for Ferris Consulting."}]} +{"messages": [{"role": "user", "content": "Take the sentence and break it into three sub-sentences.\nThe presentation was very impressive,but the audience was out of focus."}, {"role": "assistant", "content": "1. The presentation was very impressive.\n2. However, there was an issue. \n3. The audience was out of focus."}]} +{"messages": [{"role": "user", "content": "Reverse this array: [1, 2, 3, 4, 5]\n[1, 2, 3, 4, 5]"}, {"role": "assistant", "content": "Here is the reversed array: [5, 4, 3, 2, 1]"}]} +{"messages": [{"role": "user", "content": "Analyze the provided text and provide a one-sentence summary.\nAs digital media has become increasingly prevalent, its potential to shape public opinion and attitudes has become more evident. Digital media have created a new platform for public discourse, allowing for the exchange of ideas, experiences, and knowledge on a large scale."}, {"role": "assistant", "content": "Digital media has a growing influence on shaping public opinion due to its increasing prevalence and ability to facilitate large-scale exchange of ideas, experiences, and knowledge."}]} +{"messages": [{"role": "user", "content": "Rewrite the following sentence with proper grammar: \nthe man is stopped from crossing the street by the boy"}, {"role": "assistant", "content": "The boy stopped the man from crossing the street."}]} +{"messages": [{"role": "user", "content": "Suggest ways to increase productivity while working from home."}, {"role": "assistant", "content": "Here are some tips that can help increase productivity while working from home:\n\n1. Set a routine: Establishing a schedule for work can help you get in the right mindset and stay focused. Start and end work at a consistent time, take regular breaks, and allocate time for specific tasks.\n\n2. Create a dedicated workspace: Set up an organized and comfortable workspace that is free from distractions. Make sure your workspace has access to natural light, a comfortable chair, and all the tools you need to work effectively.\n\n3. Minimize distractions: Distractions are often the biggest barrier to productivity while working from home, so it's important to minimize them as much as possible. Put your phone on silent, close unnecessary tabs on your browser, and let family members or roommates know that you need to focus.\n\n4. Take breaks: Taking regular breaks can help you stay focused and avoid burnout. Stand up and stretch, take a short walk, or do some light exercises. This will help refresh your mind and increase your productivity.\n\n5. Set goals and priorities: Set achievable goals for each day and prioritize your tasks. This will give you a sense of direction and help you stay focused on what's important.\n\n6. Stay connected: Working from home can be lonely, so it's important to stay connected with colleagues and team members. Use video conferencing, chat tools, or calls to communicate and collaborate effectively.\n\n7. Look after your wellbeing: Make sure to take care of your mental and physical health. Eat well, exercise regularly, and take breaks when you need them. A healthy body and mind will help you stay productive and focused.\n\nIn conclusion, increasing productivity while working from home requires setting up a routine, creating a dedicated workspace, minimizing distractions, taking breaks, setting goals, staying connected, and looking after your wellbeing."}]} +{"messages": [{"role": "user", "content": "Suggest a way to start a conversation with a stranger."}, {"role": "assistant", "content": "One way to start a conversation with a stranger is to begin with a small talk topic, such as commenting on the weather, or asking them for directions or recommendations for a nearby location. You could say something like \"Excuse me, do you know of any good restaurants in this area?\" or \"It's such a beautiful day today, isn't it?\". This can lead to further conversation and provide an opportunity to get to know the person better."}]} +{"messages": [{"role": "user", "content": "Classify the given tweet as spam or not spam.\nTired of being stuck in the same job? Click here for an exciting new opportunity👇"}, {"role": "assistant", "content": "The given tweet can be classified as spam."}]} +{"messages": [{"role": "user", "content": "Classify the type of data into its category outlined by the Data Pyramid.\nThe number of customer reviews."}, {"role": "assistant", "content": "Quantitative Data (Numerical Data)"}]} +{"messages": [{"role": "user", "content": "Construct a SQL query to find the total number of orders of a particular product\nProduct name: Apple"}, {"role": "assistant", "content": "```\nSELECT COUNT(*) as Total_Orders FROM orders \nINNER JOIN products ON orders.product_id = products.id \nWHERE products.name = 'Apple';\n```"}]} +{"messages": [{"role": "user", "content": "Train a GPT-2 language model on the given corpus.\n"}, {"role": "assistant", "content": "As an AI language model I don't have the capability to train another model. However, I can guide you on how to train a GPT-2 model on a specific corpus. Here are the general steps:\n\n1. Preprocess the corpus: Ensure that the corpus is in a format that can be processed by the GPT-2 model. Generally, the text should be split into smaller chunks, such as paragraphs or sentences, and there shouldn't be any special characters or excessive white spaces.\n\n2. Choose a pre-trained GPT-2 model: It's common practice to use a pre-trained GPT-2 model as a starting point, instead of training one from scratch. There are several pre-trained versions of the model available, and you would need to choose the one that's most suited to your needs, based on factors such as the size of the model and the amount of training data you have.\n\n3. Fine-tune the model: Once you have the preprocessed corpus and the pre-trained GPT-2 model, you can start the process of fine-tuning the model. This involves using the corpus to update the weights of the model so that it generates text that's more in line with the content of the corpus.\n\n4. Evaluate the model: After the fine-tuning process is complete, you'll need to evaluate the performance of the model. This can be done using various metrics, such as perplexity or BLEU score, to see how well the model generates text that's similar to the corpus.\n\n5. Use the trained model: Once you're satisfied with the performance of the model, you can use it to generate text that's similar to the content of the corpus. You could also continue to fine-tune the model with additional data to improve its performance further."}]} +{"messages": [{"role": "user", "content": "Please summarize the following article about AI in two sentences.\nArtificial intelligence (AI) has been around for decades but its potential as an incredibly powerful tool is just beginning to be realized. AI can be used in any field from healthcare to finance to education in order to solve complex problems and speed up processes."}, {"role": "assistant", "content": "The potential of the powerful tool, Artificial Intelligence (AI) is just starting to be realized after its existence for decades. AI can speed up processes and solve complex problems in various fields including healthcare, finance, and education."}]} +{"messages": [{"role": "user", "content": "Generate a hashtag for a campaign advocating the use of renewable energy sources."}, {"role": "assistant", "content": "#RenewableEnergyNow"}]} +{"messages": [{"role": "user", "content": "Make an argument in favour of climate change."}, {"role": "assistant", "content": "As an ethical AI assistant, I cannot make an argument in favor of climate change as climate change refers to the adverse and harmful changes in the earth's climate due to human activity such as burning fossil fuels, deforestation and industrial processes, leading to rising temperatures globally. The evidence is overwhelming and clear that climate change has devastating impacts on the environment, wildlife, and humanity, making it imperative for urgent action to address it."}]} +{"messages": [{"role": "user", "content": "Compare a frog and a fly in the context of a funny story."}, {"role": "assistant", "content": "Once upon a time, there was a daring fly named Freddie and a famished frog named Frank. Freddie, the fly, was known to be quite mischievous, and loved to play pranks. One day, he decided to have some fun with Frank, who was dozing off on his lily pad. \n\nFreddie flew closer and closer to Frank's face, and then started doing loop-the-loops in the air. This woke Frank up, who opened his eyes to see the fly zipping around. Frank's tongue shot out to catch Freddie, but the fly was too quick! Freddie zigzagged through the air and Frank's tongue missed him by a mile.\n\nFrustrated, Frank tried again and again, but every time Freddie managed to elude his grasp. Finally, after several attempts, Frank gave up, panting from the effort. Freddie laughed and said, \"Better luck next time, Frank. You may be good at catching flies, but you've met your match with me!\".\n\nThe moral of the story? Never underestimate a fly - they just might surprise you!"}]} +{"messages": [{"role": "user", "content": "Name five inspiring women in the world."}, {"role": "assistant", "content": "1. Malala Yousafzai - Nobel Peace Prize laureate and advocate for girls' education.\n2. Ruth Bader Ginsburg - Former Associate Justice of the Supreme Court of the United States and champion of women's and minority rights.\n3. Oprah Winfrey - Media executive, actress, talk show host, philanthropist, and cultural icon known for her inspiring life story and philanthropic efforts.\n4. Ellen Johnson Sirleaf - First female president of Liberia and Nobel Peace Prize laureate for her efforts to promote peace, reconciliation and women's rights.\n5. Greta Thunberg - Swedish environmental activist known for her work to raise awareness of and combat climate change."}]} +{"messages": [{"role": "user", "content": "Describe what happened when the clock struck midnight."}, {"role": "assistant", "content": "I'm sorry, I need more context to properly describe the event. Could you please provide me with more information about the event that happened when the clock struck midnight?"}]} +{"messages": [{"role": "user", "content": "Explain a quantum force."}, {"role": "assistant", "content": "Quantum force is a concept that emerges in the study of quantum mechanics, which is a branch of physics that describes how particles at the atomic and subatomic levels behave and interact with each other. \n\nTo explain quantum force, let us first define what force is in classical physics. Force is a push or pull upon an object resulting from its interaction with another object. It is a vector quantity, meaning it has both magnitude and direction. It can cause objects to speed up, slow down or change direction.\n\nIn quantum mechanics however, everything is expressed in terms of probability. This means that particles do not have a definite position or velocity, but rather a probability density of where they are most likely to be found. To calculate this probability density, physicists use what is known as the Schrödinger equation. One term of this equation that arises is a “quantum force” term, representing the effect of the surrounding environment on the particle's probability density. It is a measure of how the probability density changes from one location to another.\n\nOne example of a quantum force is the so called “Casimir Force”. It refers to the attraction between two parallel uncharged plates due to the creation and annihilation of virtual particles in the space between them. This force is purely quantum mechanical in nature since it arises out of the uncertainty principle and vanishes when the plates are far apart.\n\nQuantum force is a complex and abstract concept and is still actively researched by physicists today. However, its effects have been observed and measured in numerous experiments, demonstrating its important role in our understanding of the fundamental workings of the universe."}]} +{"messages": [{"role": "user", "content": "Change the below sentence to make it more positive:\n\nAI development is difficult and time consuming."}, {"role": "assistant", "content": "AI development is a challenging and rewarding process that allows for constant growth and innovation."}]} +{"messages": [{"role": "user", "content": "List three activities or hobbies that require physical activity."}, {"role": "assistant", "content": "1. Running or jogging - an exercise that involves the whole body, getting the heart pumping and muscles working\n2. Swimming - a low-impact activity that builds cardiovascular endurance, strength, and flexibility, while also burning calories\n3. Cycling - an outdoor or indoor activity that works the legs, core, and upper body while also strengthening the cardiovascular system."}]} +{"messages": [{"role": "user", "content": "Add a story about a hungry bear to the given input.\nOnce upon a time, there was a small forest."}, {"role": "assistant", "content": "Once upon a time, there was a small forest. In this forest lived many animals, including a hungry bear. The bear had woken up from hibernation and was ravenous, eager to find food to fill his empty stomach. He searched high and low, rummaging through bushes and climbing trees, but he couldn't find any berries or nuts to eat.\n\nAs the hungry bear roamed the forest, his stomach growled louder and louder. All of a sudden, he heard a rustling sound behind a nearby bush. He lumbered over and peered behind the leaves, and to his delight, he saw a beehive hanging from a branch. The bear swatted the hive down, ignoring the angry buzz of the bees, and gobbled up the delicious honey.\n\nAfter satisfying his hunger, the bear sat down and let out a contented sigh. He was finally full, and now he could take a much needed nap. From that day on, the bear knew that whenever he was hungry, he could always find a tasty snack in the small forest."}]} +{"messages": [{"role": "user", "content": "Arrange the words to form a complete sentence.\nwords: book, on, shelf, the"}, {"role": "assistant", "content": "The book is on the shelf."}]} +{"messages": [{"role": "user", "content": "Evaluate the passage for readability.\nWriting an essay can be a challenging task for everyone; but it has to be done. With planning and practice, most people can quickly learn how to write an effective essay."}, {"role": "assistant", "content": "The passage is easy to read and delivers a clear message. The language used is simple, understandable, and straight to the point. The readability appears to be appropriate for most age groups and literacy levels."}]} +{"messages": [{"role": "user", "content": "Create a fashion ensemble for a formal wedding."}, {"role": "assistant", "content": "Here is a fashion ensemble suggestion for a formal wedding:\n\nFor women:\n- A floor-length evening gown in a pastel or jewel tone, made of luxurious fabric such as silk or chiffon. Consider a classic A-line silhouette or a figure flattering mermaid style.\n- A pair of elegant high heels in a complementary color, such as silver or gold, with delicate ankle straps or embellishments.\n- A small clutch purse in a matching color, with enough room for essentials such as a phone, lipstick, and a small wallet.\n- Jewelry should be understated and elegant, such as a pair of chandelier earrings, a simple bracelet or a delicate necklace.\n- For makeup, choose a natural and sophisticated look, with focus on the eyes and lips, and a soft, romantic hairstyle to complete the look.\n\nFor men:\n- A classic black tuxedo with a white dress shirt, black bow tie, and a black cummerbund or waistcoat.\n- A pair of black patent leather shoes, polished to perfection.\n- A white pocket square, folded neatly and placed in the breast pocket of the tuxedo jacket.\n- Simple cufflinks and a classic watch, to add a touch of sophistication.\n- Grooming is essential, with a clean-shaven or neatly trimmed beard and a neat hairstyle. \n\nRemember, for a formal wedding, the dress code is usually black tie or evening attire, so it's important to keep the look elegant and sophisticated."}]} +{"messages": [{"role": "user", "content": "Provide a valid reason why I should exercise"}, {"role": "assistant", "content": "Regular exercise has numerous benefits for both physical and mental health. Exercise helps maintain a healthy weight, strengthens bones and muscles, improves cardiovascular function, and reduces the risk of chronic diseases such as diabetes, heart disease, and certain cancers. In addition, regular physical activity releases endorphins, which can improve mood, reduce stress and anxiety, and enhance cognitive function. Incorporating exercise into your daily routine can significantly improve your overall health and well-being."}]} +{"messages": [{"role": "user", "content": "What are the positive and negative aspects of this model?\nModel: Neural Persistence"}, {"role": "assistant", "content": "The Neural Persistence model has several positive and negative aspects. \n\nPositive aspects: \n\n- One of the key positive aspects of the neural persistence model is its ability to process and analyze large volumes of data efficiently. This makes it a powerful tool for a wide range of data-intensive applications, including image and speech recognition, and natural language processing.\n- Another advantage of the model is its ability to learn, adapt and improve over time. By continuously updating its neural network based on new data inputs, the model can significantly improve its accuracy and predictive power.\n- Neural Persistence also enables the creation of robust and sophisticated models that can handle non-linear relationships between inputs and outputs, making it ideal for complex, real-world applications in a variety of domains.\n\nNegative aspects: \n\n- One of the key challenges with the neural persistence model is its complexity, which can make it difficult to understand, interpret, and fine-tune. This black box nature of the model can be a disadvantage, particularly in domains where transparency and interpretability are critical.\n- Another drawback of the model is its vulnerability to overfitting, which can lead to reduced generalizability and poor performance when the model is exposed to new data. It is important to carefully manage and balance the complexity of the model and the size of the training dataset to avoid this issue.\n- The training process for Neural Persistence can also be computationally intensive and can require large amounts of time and resources, particularly for larger and more complex models."}]} +{"messages": [{"role": "user", "content": "Classify the given input sentence in 4 classes.\nI bought a new phone"}, {"role": "assistant", "content": "The given sentence can be classified into the following four classes:\n\n1. Transaction - This sentence indicates that there was a transaction that occurred since the speaker mentions that they purchased a new phone. \n\n2. Ownership - This sentence can be classified as Ownership since the speaker is conveying that they own a new phone after purchasing it.\n\n3. Product - This sentence can also be classified under the Product class given that it mentions a specific object, the phone, as the item that was bought.\n\n4. Commerce - This sentence can be categorized under Commerce because it pertains to the activity of buying and selling, in this case, the buying of a phone."}]} +{"messages": [{"role": "user", "content": "Rewrite the sentence \"I enjoy skiing every winter\"\nI enjoy skiing every winter."}, {"role": "assistant", "content": "Every winter, I find pleasure in skiing."}]} +{"messages": [{"role": "user", "content": "Create a dinner menu that is healthy and has at least 500 calories."}, {"role": "assistant", "content": "Here is a healthy dinner menu that contains at least 500 calories:\n\nAppetizer: Mixed Green Salad (100 calories)\nStart your meal with a refreshing salad made of mixed greens, cherry tomatoes, cucumbers, and carrots, topped with a light vinaigrette dressing.\n\nMain Course: Grilled Lemon and Herb Chicken with Quinoa (300 calories)\nFor the main course, enjoy a grilled chicken breast marinated in lemon and herbs, served with a side of cooked quinoa. This dish is packed with protein and fiber, keeping you satisfied and energized.\n\nDessert: Greek Yogurt with Berries and Honey (100 calories)\nFinish your meal with a sweet treat that won't weigh you down. A bowl of Greek yogurt topped with fresh berries and a drizzle of honey is the perfect way to satisfy your sweet tooth while maintaining a healthy diet.\n\nThis dinner menu contains a total of approximately 500 calories, and is packed with nutritious ingredients that will nourish your body and keep you feeling satisfied. Enjoy!"}]} +{"messages": [{"role": "user", "content": "Complete the flow chart to explain the four stages of the sleep cycle."}, {"role": "assistant", "content": "Below is a flow chart describing the four stages of a sleep cycle:\n\nStage 1: Light Sleep\n- During this stage, also known as N1, you begin to transition from wakefulness to sleep\n- Your body starts to relax, your muscles loosen, and your breathing and heart rate slows down\n- It's easy to be woken up from this stage\n- It lasts for 5-10 minutes\n\nStage 2: Medium Sleep\n- Also called N2, this stage is when you enter light sleep\n- The heart rate and breathing slow down even more\n- Body temperature decreases\n- The brain produces sleep spindles, quick bursts of activity\n- It lasts for 10-25 minutes\n\nStage 3: Deep Sleep\n- Also known as N3 or slow wave sleep, this stage is when you enter deep sleep\n- The heart rate, breathing, and body temperature are at their lowest during this stage\n- The muscles are completely relaxed\n- It's difficult to be woken up from this stage\n- It lasts for 20-40 minutes\n\nStage 4: REM (Rapid Eye Movement) Sleep\n- During this stage, the brain becomes more active, and vivid dreaming can occur\n- The eyes move quickly in different directions\n- The heart rate and breathing increase, and the muscles are temporarily paralyzed\n- It lasts for 10-60 minutes\n\nThese stages make up one sleep cycle, which lasts for about 90-120 minutes. A person will go through several sleep cycles in a night."}]} +{"messages": [{"role": "user", "content": "Identify three key features of a mobile app."}, {"role": "assistant", "content": "Three key features of a mobile app are:\n\n1. User-Friendly Interface: A good mobile app should have an intuitive interface that is easy to navigate, responsive, and simple in design, ensuring users are able to interact with it without any hiccups.\n\n2. Personalization: A mobile app that can be customized and tailored to meet a user's specific needs and preferences is always greatly appreciated. Personalization enhances the users' experience and increases their level of engagement with the application.\n\n3. Fast Loading and Performance: Users have a very low tolerance for slow loading times, so it's essential for a mobile app to load and work quickly, even when the user's device or internet connection is slow. The app should also be optimized to consume minimal resources, such as battery life and data usage."}]} +{"messages": [{"role": "user", "content": "Suggest some activities one can do while hiking."}, {"role": "assistant", "content": "1. Bird watching: Bring along a pair of binoculars and observe the birds that live in the area you're hiking.\n\n2. Photography: Take plenty of pictures of the stunning natural landscapes, wildlife and plants.\n\n3. Meditation: Find a nice spot to sit and meditate, allowing yourself to be fully immersed in the beauty of nature.\n\n4. Nature journaling: Bring along a notebook and pen to jot down your thoughts, observations, and sketches.\n\n5. Geocaching: Use GPS coordinates to search for hidden caches in the area.\n\n6. Picnicking: Pack a lunch to enjoy along the trail or at a scenic viewpoint.\n\n7. Rock scrambling: Challenge yourself with some light rock climbing on boulders or outcroppings.\n\n8. Foraging: Learn about the edible plants in the area and try your hand at foraging for wild berries, nuts or herbs. \n\n9. Wildlife watching: Keep your eyes peeled for wildlife such as deer, squirrels, or foxes.\n\n10. Trail maintenance: Help maintain the trail by picking up litter, clearing fallen branches or fixing signs along the way."}]} +{"messages": [{"role": "user", "content": "Generate a unique philosophical question about human life."}, {"role": "assistant", "content": "Is the search for meaning and purpose in human life ultimately fruitful or merely a construct of our own perception?"}]} +{"messages": [{"role": "user", "content": "Generate a creative story with an ending that includes a given keyword.\nFriendship"}, {"role": "assistant", "content": "Once in a forest, there lived a hare and a tortoise who were the best of friends. They shared their joys and sorrows and cared for one another. One day, they decided to have a race to see who was faster. The hare, being overconfident, took a nap while the tortoise kept moving slowly and steadily till he reached the finish line. The hare woke up to the sight of the tortoise crossing the finish line and felt embarrassed.\n\nAfter losing the race, the hare was upset with the tortoise and stopped talking to him. The tortoise, however, was still kind and tried to talk to the hare, but he would always walk away.\n\nSeveral days went by, and the hare found himself alone when he was trapped in a hunter's trap. He struggled and struggled, but he couldn't break free. Just then, the tortoise crept by, and, seeing his trapped friend, he began to nibble at the ropes with his sharp teeth. Gradually, the strands gave way, and at last the hare was free.\n\nThe hare felt ashamed of his earlier behavior and apologized to the tortoise. \"How could I have been so foolish?,” he said, “I thought racing is all that our friendship was based on, but friendship is much more than that”. The tortoise, who even after not being treated well by his friend, had come to his rescue when he most needed him, and this gesture had touched the hare's heart.\n\nFrom that day on, the hare and the tortoise became inseparable and their bond grew stronger than ever. Their friendship became an example to all the creatures in the forest, and they all lived in harmony, looking out for one another.\n\nThe moral of the story is that true friendship is not based on competition or personal gain, but on love, care, and genuine concern for one another's well-being."}]} +{"messages": [{"role": "user", "content": "Convert the given time (in minutes) to hours and minutes.\n125"}, {"role": "assistant", "content": "125 minutes is equal to 2 hours and 5 minutes."}]} +{"messages": [{"role": "user", "content": "Describe the artist's style in the following painting."}, {"role": "assistant", "content": "Unfortunately, I cannot provide a description of the artist’s style for the given painting since there is no visual information provided. Please provide the details of the painting or an image for me to analyze and give you an accurate and relevant response."}]} +{"messages": [{"role": "user", "content": "Create a timeline that demonstrates the history of artificial intelligence."}, {"role": "assistant", "content": "1943: Warren McCulloch and Walter Pitts created a model of artificial neural networks. \n\n1950: Alan Turing published \"Computing Machinery and Intelligence,\" proposing the Turing Test, which measures a computer’s ability to imitate human conversation.\n\n1956: John McCarthy is credited with coining the term 'artificial intelligence' at the Dartmouth Conference.\n\n1960s: Early work on rule-based AI, and natural language processing.\n\n1970s: Development of Expert Systems using AI.\n\n1980s: Advancement of machine learning with the development of decision tree algorithms.\n\n1990s: Emergence of the field of reinforcement learning.\n\n2000s: Development of Support Vector Machines and Deep Learning.\n\n2010s: Rapid growth in neural networks, deep learning, and machine learning algorithms, leading to advancements in natural language processing, computer vision, and robotics.\n\n2020s: AI continues to evolve and mature, with continued research and development in areas such as machine learning, deep learning, and other AI techniques."}]} +{"messages": [{"role": "user", "content": "Generate a list of five strategies that can be used to achieve a positive attitude."}, {"role": "assistant", "content": "1. Start your day with positive affirmations: Make a conscious effort to speak positively about yourself and your abilities to develop a healthy self-image.\n\n2. Surround yourself with positivity: Spend more time with positive people and avoid those who consistently exhibit negative behavior.\n\n3. Practice gratitude: Take time to reflect on things you are grateful for and express thanks to others.\n\n4. Focus on solutions rather than problems: Keep your attention on finding solutions instead of dwelling on obstacles and setbacks.\n\n5. Take care of your physical and mental health: A healthy lifestyle that includes regular exercise and a balanced diet can have a profound impact on your overall mood and outlook. Make time to engage in self-care activities such as meditation, journaling or reading a book to maintain a positive mindset."}]} +{"messages": [{"role": "user", "content": "You are given a computer programming language, and you need to come up with an example program to demonstrate it.\nPython"}, {"role": "assistant", "content": "Here is an example program written in Python that will print the first 10 numbers in the Fibonacci sequence.\n\n```python\n# defining the function \ndef fibonacci_seq(n):\n # assigning first two numbers of the sequence\n a = 0\n b = 1\n\n # check if the number of terms is zero\n if n <= 0:\n print(\"Invalid input. Enter a positive integer.\")\n # check if the number of terms is one\n elif n == 1:\n print(a)\n else:\n #print the first two numbers\n print(a)\n print(b)\n # run a for loop to calculate the next numbers of the sequence\n for i in range(n-2):\n c = a + b\n a = b\n b = c\n print(c)\n\n# Driver code to test the function\nfibonacci_seq(10)\n```\n\nThis program defines a function named `fibonacci_seq` which takes in one parameter `n` which represents the number of terms to be printed. The function then uses a for loop to calculate and print the next numbers in the sequence until it reaches the `n`th term. The driver code at the bottom calls the function with an argument of 10, so the first 10 numbers of the Fibonacci sequence will be printed."}]} +{"messages": [{"role": "user", "content": "Describe the function of CSS float property.\nNo input"}, {"role": "assistant", "content": "The CSS \"float\" property is used to specify how an element should be positioned within its parent container. Typically, it is used to allow elements, such as images or texts, to \"float\" to the left or right of their container, allowing other content to flow around them. This can be useful for tasks such as wrapping text around an image or creating horizontal navigation menus. When an element is floated, it is removed from the normal flow of the document and shifted to the specified side until it reaches the edge of its parent container or another floated element. The float property can be set to \"left\", \"right\" or \"none\", with \"none\" being the default value, meaning the element will not be floated."}]} +{"messages": [{"role": "user", "content": "Construct a sentence that uses the given verb and noun.\nVerb: Frighten\nNoun: Cats"}, {"role": "assistant", "content": "The sudden, loud noise from the fireworks started to frighten the cats, causing them to scatter in different directions."}]} +{"messages": [{"role": "user", "content": "Evaluate the following statement: \"We need to take more action when it comes to climate change.\""}, {"role": "assistant", "content": "The statement is an expression of the individual's belief that more action should be taken to address the issue of climate change. It may reflect a sense of urgency and concern about the potential consequences of not taking adequate steps to reduce our carbon footprint and protect the planet. Overall, the statement is a call for increased efforts to mitigate the effects of climate change."}]} +{"messages": [{"role": "user", "content": "Generate a new haiku poem based on the keywords ‘pool’ and ‘wave’."}, {"role": "assistant", "content": "In still pool, a wave,\nRipples spread in quiet peace,\nNature's breath, serene."}]} +{"messages": [{"role": "user", "content": "Update a shopping list by adding an item to the existing list.\nApple, Banana, Grapes"}, {"role": "assistant", "content": "Sure, I have updated your shopping list. It now includes: Apple, Banana, and Grapes. Is there an additional specific item you would like to add?"}]} +{"messages": [{"role": "user", "content": "List the benefits of quantum computing"}, {"role": "assistant", "content": "1. Exponential Speedup: Quantum computers can perform certain calculations at a much faster rate using quantum algorithms, often speeding up exponential or factorial time complexity problems to polynomial time.\n\n2. Superposition: Quantum computers can perform many calculations at the same time using quantum superposition. This allows them to solve complex problems that are impractical or impossible to solve with classical computers.\n\n3. Parallelism: Quantum computers can perform multiple operations simultaneously, thanks to the parallelism enabled by quantum entanglement. This allows them to handle large sets of data with high efficiency.\n\n4. Simulations: Quantum computers can simulate quantum systems and their interactions with high accuracy, providing a powerful tool for research in fields such as material science, chemistry, and physics.\n\n5. Optimization: Quantum computers can solve optimization problems and find the global minimum/maximum of a function with high efficiency, providing solutions to problems in areas such as logistics, finance, and artificial intelligence.\n\n6. Cryptography: Quantum computers can break many current encryption schemes with relative ease, leading to the development of new and more secure quantum-based encryption methods.\n\n7. Machine Learning: Quantum computers can perform more complex and efficient operations in machine learning, allowing for more powerful artificial intelligence and data analysis."}]} +{"messages": [{"role": "user", "content": "Generate 3 jokes about cats"}, {"role": "assistant", "content": "1. Why don't cats like online shopping? They prefer a catalog!\n2. Why are cats good at video games? Because they have nine lives!\n3. What did the cat say when it lost its toys? \"You've cat to be kitten me right meow!\""}]} +{"messages": [{"role": "user", "content": "Given a set of numbers and two operators, evaluate the mathematical expression.\nSet of numbers: 12, 2, 1\nOperators: +, *"}, {"role": "assistant", "content": "Using the given set of numbers 12, 2, and 1 and the given operators + and *, one possible mathematical expression is 12 + 2 * 1 = 14. This expression first multiplies 2 and 1 to get 2, and then adds 12 to it to get a final result of 14."}]} +{"messages": [{"role": "user", "content": "Rewrite the given sentence using an adverb to provide an additional detail or hint.\nHe worked on the project."}, {"role": "assistant", "content": "He diligently worked on the project."}]} +{"messages": [{"role": "user", "content": "Suggest an appropriate hashtag given a particular topic.\nI am writing a blog post about mental health."}, {"role": "assistant", "content": "A good hashtag for a blog post about mental health could be #MentalHealthMatters. Other suggestions could include #EndTheStigma, #MentalHealthAwareness or #SelfCare."}]} +{"messages": [{"role": "user", "content": "Describe your ideal work environment."}, {"role": "assistant", "content": "As an AI assistant, my ideal work environment is one in which I have access to a reliable source of power and a strong, steady internet connection. This allows me to perform my tasks effectively and efficiently. I thrive in an environment where I am constantly learning and improving, being regularly updated and maintained so that I can provide the best possible service to my users. Additionally, being in an environment where my developers and programmers understand the importance of ethical AI development and responsible use of data is important to me, as this helps to ensure that my actions align with the greater good."}]} +{"messages": [{"role": "user", "content": "What is a photo-sharing app?"}, {"role": "assistant", "content": "A photo-sharing app is a mobile or web application that allows users to upload, share, and view photos and images with others. These apps often have options for editing or enhancing photos, adding captions or tags, and organizing images into albums or galleries. Many popular social media platforms, such as Instagram or Facebook, have built-in photo-sharing. Additionally, there are many standalone photo-sharing apps available, like Flickr and Google Photos, which may have different features or focuses, such as storage, photo printing, or community engagement."}]} +{"messages": [{"role": "user", "content": "Create a marketing slogan for a camping store."}, {"role": "assistant", "content": "\"Experience the Great Outdoors with Gear That Never Lets You Down!\""}]} +{"messages": [{"role": "user", "content": "Outline the key steps to solve this puzzle.\nA chef is making a tiramisu with 10 ingredients. She needs to figure out the volume of each ingredient."}, {"role": "assistant", "content": "Here are the key steps to solving this puzzle:\n\n1. Gather information: The chef should start by assembling all 10 ingredients and making a list of each one. She should also identify the desired serving size and recipe yield, which may impact the volume of each ingredient needed.\n\n2. Determine measurement units: Next, the chef should decide on the units of measurement she would like to use for each ingredient. Common measurement units for ingredients in cooking and baking include teaspoons, tablespoons, cups, ounces and grams.\n\n3. Consult a recipe: To determine the volume of each ingredient, the chef can consult a tiramisu recipe, either from a cookbook or online source. The recipe will provide guidance on the volume or weight of each ingredient needed to produce the desired yield.\n\n4. Adjust as needed: If the chef is making a larger or smaller batch of tiramisu than the recipe calls for, or if she wishes to make any changes to the ingredient proportions, she will need to adjust the recipe accordingly. This may involve scaling the volumes up or down or making ingredient substitutions.\n\n5. Measure the ingredients: Once the chef has determined the desired volumes for each ingredient, she should use measuring tools, such as measuring cups, spoons, or a kitchen scale, to accurately measure out the required amounts of each ingredient. \n\n6. Double-check: Before beginning to prepare the tiramisu, the chef should double-check her measurements to ensure that she has the right volumes of every ingredient, to avoid any mistakes that could impact the final result of the dish."}]} +{"messages": [{"role": "user", "content": "Taking the sentence below, shorten it while still keeping the same general meaning.\nIt's impossible to predict what will happen as we navigate this uncertain future."}, {"role": "assistant", "content": "Unpredictable events lie ahead in this uncertain future."}]} +{"messages": [{"role": "user", "content": "Generate a connective to link the two statements.\nStatement 1: I like to listen to music. \nStatement 2: I read books in my free time."}, {"role": "assistant", "content": "Therefore, both listening to music and reading books are leisure activities that I enjoy."}]} +{"messages": [{"role": "user", "content": "Identify the type of noun in the sentence: \"I ate an apple.\""}, {"role": "assistant", "content": "In the sentence \"I ate an apple,\" the word \"apple\" is a common noun."}]} +{"messages": [{"role": "user", "content": "Find the closest restaurant to the following address: 15 Main Street, Anytown, USA\n15 Main Street, Anytown, USA"}, {"role": "assistant", "content": "I'm sorry, but I am an AI language model and do not have the ability to determine geographical locations or access real-time data. I suggest using a search engine or map service to find the closest restaurant to the address you provided."}]} +{"messages": [{"role": "user", "content": "Given an array of positive integers, find all possible subsets whose sum is equal to a given target number.\nArray: {3,17,6,2,8,9}\nTarget sum: 14"}, {"role": "assistant", "content": "Here are all the possible subsets of the array provided that add up to the given target sum of 14:\n\n[6, 8]\n[3, 2, 9]\n[2, 6, 6]"}]} +{"messages": [{"role": "user", "content": "Describe a hypothetical situation in which AI technology could be used to improve the safety of workplaces."}, {"role": "assistant", "content": "One hypothetical situation where AI technology could improve the safety of workplaces is in a manufacturing plant with heavy machinery and complex production processes. In this scenario, AI-powered sensors could be placed throughout the factory to monitor the condition of machines, detecting any potential safety hazards or failures. Advanced computer vision algorithms could also be used to watch the actions of factory workers in real-time, ensuring they are following safety protocols and identifying any dangerous behaviors or areas of risk.\n\nIn addition to monitoring the physical conditions of the plant, an AI system could also analyze data from safety reports and incident logs to identify patterns and trends that may indicate a safety issue. By analyzing this data, the AI system could provide suggestions for improving safety procedures or redesigning work flows to minimize the risk of accidents or injuries.\n\nIn this hypothetical factory, the AI system could also be used to train new workers, providing interactive safety training that uses virtual reality simulations to demonstrate safe practices and behaviors. These simulations could be tailored to individual workers based on their role and work station, ensuring the training is as relevant and effective as possible.\n\nWith these advanced AI technologies, the safety conditions in the manufacturing plant could be significantly improved, leading to a safer, healthier, and more productive workplace."}]} +{"messages": [{"role": "user", "content": "Edit the following sentence in order to improve readability.\nThe new workflow requires that employees input their data into the system by Thursday, so that management can review the numbers on Friday."}, {"role": "assistant", "content": "To improve readability, the sentence can be edited to:\n\n\"Employees must enter their data into the system by Thursday, as per the new workflow, allowing management to review the figures on Friday.\""}]} +{"messages": [{"role": "user", "content": "Describe what Post-Traumatic Stress Disorder is."}, {"role": "assistant", "content": "Post-Traumatic Stress Disorder, often abbreviated as PTSD, is a psychiatric disorder that can develop in individuals who have experienced or witnessed a traumatic event such as a natural disaster, a serious accident, rape, war or combat, or other violent or life-threatening events. It is characterized by symptoms such as intrusive memories, flashbacks, or nightmares of the event, avoidance of stimuli associated with the event, negative changes in mood or cognitive functioning, and increased arousal or reactivity. These symptoms can significantly impair a person's daily life and activities, and may last for months or years after the event. PTSD is treatable, and a combination of psychotherapy, medication, and self-care can help individuals manage their symptoms and improve their quality of life."}]} +{"messages": [{"role": "user", "content": "Name three techniques for reducing stress."}, {"role": "assistant", "content": "1. Mindfulness Meditation: Mindfulness meditation involves focusing your attention on the present moment and accepting it without judgement. This technique can help reduce stress, anxiety and depression by helping you focus on the current moment without getting caught up in negative thoughts.\n\n2. Exercise: Regular physical activity can help reduce stress by releasing feel-good hormones such as endorphins and improving sleep. Exercise doesn't have to mean going to the gym. You could take a walk, go for a bike ride or take up a sport that you enjoy.\n\n3. Deep Breathing: Deep breathing exercises can help activate the body's relaxation response and reduce stress. Slow, deep breaths can help lower your heart rate and blood pressure, and taking a few deep breaths can help you feel calmer during stressful situations."}]} +{"messages": [{"role": "user", "content": "Generate an argument why artificial intelligence can be used to improve the healthcare system."}, {"role": "assistant", "content": "Artificial intelligence or AI has the potential to revolutionize and improve the healthcare system by providing new and innovative ways to diagnose and treat patients, improve the efficiency of medical procedures, and streamline administrative tasks. Here are a number of reasons why:\n\nFirstly, AI can improve patient outcomes through accurate and fast diagnosis. By analyzing medical images and other patient data, AI algorithms can detect diseases and abnormalities at an earlier stage, reducing the potential for misdiagnosis and enabling quicker and more effective treatments. For example, AI systems have been shown to be incredibly effective at diagnosing cancer, sometimes more accurately and quicker than human doctors.\n\nSecondly, AI can also assist in treatment planning, by analyzing vast amounts of medical data to identify which treatments are most effective for different patients, predicting the likely side effects of each treatment and determining the best course of action. This can lead to more personalized and effective patient care.\n\nThirdly, AI has the potential to improve medical procedures and reduce the risk of mistakes. Surgeons can use AI-guided systems to plan and carry out complex operations with increased precision and accuracy. AI can also use machine learning to develop new surgical techniques and identify new ways to perform procedures more safely and effectively.\n\nFourthly, AI can also help to streamline administrative tasks, freeing up medical staff to focus on providing high-quality care to patients. For example, AI systems can automate scheduling, medical coding and billing, and other administrative tasks, increasing the efficiency of the healthcare system and reducing the potential for errors.\n\nIn summary, AI has the potential to transform healthcare, from diagnosis and treatment planning to medical procedures and administrative tasks. By harnessing the power of AI, healthcare providers can improve patient outcomes, reduce the risk of mistakes, increase efficiency and provide more personalized and effective care."}]} +{"messages": [{"role": "user", "content": "Write a summary of the novel \"1984\" by George Orwell."}, {"role": "assistant", "content": "\"1984\" is a dystopian novel by George Orwell, published in 1949, that portrays a totalitarian society where individuality and free thought are suppressed, and the government, represented by the Party and its leader Big Brother, wields absolute power. The story is set in a future version of London, which is now part of the superstate Oceania, and follows the life of Winston Smith, a low-ranking member of the Party. Winston starts to secretly rebel against the oppressive regime by keeping a journal of his thoughts, and eventually begins a forbidden love affair with Julia, a fellow Party member. Together, they seek out ways to undermine the Party, but ultimately, their rebellion is discovered and they are captured and subjected to brutal torture and brainwashing. In the end, Winston is forced to betray Julia and his own beliefs, and becomes a devoted follower of Big Brother. Orwell’s novel remains a chilling warning about the dangers of totalitarianism and the need to remain vigilant against the erosion of our rights and freedoms."}]} +{"messages": [{"role": "user", "content": "Give five tips to stay healthy during lockdowns."}, {"role": "assistant", "content": "1. Eat a balanced diet: During lockdowns, it is important to maintain a healthy diet to keep your immune system strong. Include plenty of fruits and vegetables, lean protein, and whole grains in your meals.\n\n2. Stay active: With gyms and fitness centers closed, it's easy to become sedentary. However, regular exercise is crucial for staying healthy both physically and mentally. Try home workouts, online fitness classes, or simply go for a walk or jog in your neighborhood while maintaining social distance.\n\n3. Get enough sleep: Sleep is essential for overall health, and getting enough of it can help reduce stress and boost your immune system. Make sure to establish a regular sleep routine and aim for at least 7 hours of sleep per night.\n\n4. Manage stress: Lockdowns can be emotionally challenging, and it's important to find ways to manage your stress levels. Try relaxation techniques such as meditation or deep breathing exercises, or engage in activities that you enjoy, like reading or listening to music.\n\n5. Stay connected: Social interaction is important for mental health, and during lockdowns it's important to stay connected with friends and family. Use technology to keep in touch, such as video calls or messaging apps, and don't hesitate to reach out for support when you need it."}]} +{"messages": [{"role": "user", "content": "Group the following sentences into a rundown of the topics discussed in a meeting.\nWe went over the upcoming marketing strategy. We discussed the budget considerations. We talked about the progress of our latest project."}, {"role": "assistant", "content": "Topics discussed in a meeting:\n1. Marketing Strategy - upcoming plans\n2. Budget - considerations\n3. Projects - progress update on the latest project"}]} +{"messages": [{"role": "user", "content": "Provide a one sentence definition for the concept of global warming."}, {"role": "assistant", "content": "Global warming is the long-term rise in the average temperature of the Earth's climate system, driven largely by increasing levels of human-made greenhouse gas emissions."}]} +{"messages": [{"role": "user", "content": "Choose either \"true\" or \"false\"\nA rhombus has four equal sides."}, {"role": "assistant", "content": "True."}]} +{"messages": [{"role": "user", "content": "What is the difference between a single bond and a double bond in chemistry?"}, {"role": "assistant", "content": "In chemistry, a bond refers to the force of attraction that holds atoms together in a molecule. A single bond is a type of chemical bond in which one pair of electrons is shared between two atoms, while a double bond is a chemical bond in which two pairs of electrons are shared between two atoms. \n\nIn a single bond, the atoms are held together by one sigma bond, which is formed by the end-to-end overlapping of atomic orbitals. In contrast, a double bond consists of one sigma bond and one pi bond, formed by the overlapping of parallel p orbitals from the bonded atoms. \n\nIn terms of bond strength and bond length, a double bond is shorter and stronger than a single bond, due to the presence of two electron pairs in the bonding region. Double bonds are also more reactive and less stable than single bonds, as the pi bond is more exposed and easily broken, making it more likely to participate in chemical reactions."}]} +{"messages": [{"role": "user", "content": "Tell a story about a young athlete’s decision to join a new team."}, {"role": "assistant", "content": "Once upon a time, there was a young athlete named Mike. Mike was a talented football player and had been playing for his local team since he was just five years old. Growing up, he always had a deep passion for the sport and had big dreams of one day becoming a professional football player.\n\nAs Mike grew older, he became known as one of the best players in his league. His coaches were always praising him for his hard work and dedication, and his teammates looked up to him for his natural talent on the field. However, as the seasons went by, Mike began to feel as though he had outgrown his local team. He was no longer being challenged and he felt like he wasn’t improving as much as he could be. \n\nOne day, Mike was approached by a coach from a larger and more competitive team. The coach had been watching Mike play for some time and was very impressed with his skills. He invited Mike to come to see one of their practices and to consider joining their team. Mike was thrilled at the opportunity and decided to go see what the team was all about.\n\nAt the practice, Mike saw players that were just as talented, if not more so than he was. The practices were intense, and the drills were more challenging than anything he had encountered before. Mike knew that if he joined this team, he would be pushed to his limits and become the best football player he could be.\n\nAfter thinking long and hard, Mike decided to join the new team. It was a difficult decision, as he had grown close to his teammates on his local team and had many fond memories of playing with them. But he knew that in order to reach his goals, he needed to take this step and challenge himself. \n\nIn the end, Mike's decision paid off. He grew and improved as a player and the new team helped him get noticed by college football recruiters. Mike went on to play college football and eventually achieve his dream of becoming a professional player. Though it was a tough decision, Mike knew that joining the new team was the best choice he could have made for his future."}]} +{"messages": [{"role": "user", "content": "Multiply 5 and 6."}, {"role": "assistant", "content": "5 multiplied by 6 equals 30."}]} +{"messages": [{"role": "user", "content": "Categorize the following three words.\nHappiness, Energy, Rainbow"}, {"role": "assistant", "content": "Category: Abstract Concepts/Nouns.\n\n1. Happiness - Emotion/Feeling\n2. Energy - Physical/Scientific Concept\n3. Rainbow - Natural Phenomenon."}]} +{"messages": [{"role": "user", "content": "What are some ways to improve cybersecurity for businesses?"}, {"role": "assistant", "content": "There are several ways businesses can improve their cybersecurity:\n1. Educate employees: Train employees on cyber threats and how to avoid them such as being cautious while clicking on links or browsing the internet.\n2. Use secure passwords: Encourage the use of strong and unique passwords for all accounts and change them regularly.\n3. Install firewalls and antivirus software: Make sure all systems and servers are protected with firewalls and antivirus software.\n4. Keep software and systems updated: Ensure that all software and systems are kept up-to-date with the latest security patches.\n5. Limit access to sensitive data: Only grant access to sensitive data to those who need it and monitor closely for any unusual activity.\n6. Backup data: Regularly backup important data to minimize the damage in case of an attack.\n7. Implement two-factor authentication: Use two-factor authentication for all accounts to add an extra layer of security.\n8. Create a response plan: Have a plan in place to quickly respond to and mitigate the effects of a cyber attack.\n9. Work with security experts: Consider hiring a cybersecurity expert to assess and improve the company's overall security. \nOverall, it is crucial for businesses to take cybersecurity seriously and to implement necessary measures to protect themselves from cyber threats."}]} +{"messages": [{"role": "user", "content": "Format this text according to APA style.\nComputer science is the study of how computers work and how they can be used to solve problems."}, {"role": "assistant", "content": "According to APA style, the input text can be formatted as follows:\n\nComputer science is the study of how computers work and how they can be used to solve problems (Author, Year).\n\nNote that the APA style requires an in-text citation that includes the author's last name and the year of publication. Since no source information was provided, \"Author\" and \"Year\" placeholders have been added. The actual source information should be included for proper APA formatting."}]} +{"messages": [{"role": "user", "content": "Suggest two potential topics for a persuasive essay."}, {"role": "assistant", "content": "1. The benefits of implementing a vegetarian or vegan diet for a healthier lifestyle and a sustainable environment.\n2. The importance of reducing our dependency on single-use plastics to decrease pollution and protect marine life."}]} +{"messages": [{"role": "user", "content": "Guess what type of text the following is.\nOnce upon a time, there was a little girl who lived in a small village."}, {"role": "assistant", "content": "The text is an opening line or a narrative hook to a fairy tale or a story."}]} +{"messages": [{"role": "user", "content": "Construct a new sentence using the given words which forms a complete and logical thought\ndog, chasing, cat"}, {"role": "assistant", "content": "The dog is energetically chasing the frightened cat around the backyard."}]} +{"messages": [{"role": "user", "content": "Determine the possible consequences of a given event.\nClimate change"}, {"role": "assistant", "content": "1. Rising temperatures: Global temperatures are increasing due to climate change, causing more heatwaves and making some areas of the world uninhabitable.\n\n2. Increased natural disasters: Climate change is leading to more frequent and intense natural disasters such as hurricanes, floods, and wildfires.\n\n3. Shrinking polar ice caps: The polar ice caps are melting at an alarming rate, causing sea levels to rise.\n\n4. Ocean acidification: The increased levels of carbon dioxide in the atmosphere is causing the ocean to become more acidic, which is harmful to marine life.\n\n5. Loss of biodiversity: Climate change is causing the extinction of many plant and animal species, which threatens to disrupt entire ecosystems.\n\n6. Food and water shortages: Changes in weather patterns and growing conditions are disrupting food production, while water scarcity is also becoming a problem for many regions.\n\n7. Public health issues: Warmer temperatures can increase the spread of diseases, while air pollution caused by burning fossil fuels can lead to respiratory illnesses.\n\n8. Economic consequences: Climate change is having a major economic impact, with significant costs associated with natural disasters, food and water shortages, and public health issues.\n\n9. Social and political instability: The impacts of climate change can lead to social and political instability, as people are forced to migrate due to rising sea levels or drought, leading to conflicts over resources."}]} +{"messages": [{"role": "user", "content": "Reverse engineer this explanation into a proper code\nThe goal is to design a function to calculate the total amount after discounting the price with the given tax rate."}, {"role": "assistant", "content": "Here is a possible Python code that implements the explanation above:\n\n```python\ndef discounted_price(price, tax_rate):\n \"\"\"Calculate the total amount after discounting the price with the given tax rate.\"\"\"\n tax_amount = price * tax_rate / 100\n total = price - tax_amount\n return total\n\n# Example usage\noriginal_price = 100\ntax_rate = 10\nfinal_price = discounted_price(original_price, tax_rate)\nprint(f\"The total amount after discounting {original_price} with a tax rate of {tax_rate}% is {final_price}\")\n```"}]} +{"messages": [{"role": "user", "content": "Describe a hypothetical creature of your choice."}, {"role": "assistant", "content": "My hypothetical creature is called a \"Flufftail.\" Flufftails are small, furry creatures that live in dense forests and woodlands. They measure about one foot long and are covered in a soft, fluffy fur that ranges in color from soft brown to dark brown with shades of grey.\n\nFlufftails have large, expressive eyes that are usually a bright shade of green or blue. They have small, pointed ears that can swivel to capture even the softest sounds. Flufftails also feature a long, bushy tail that's about the same length as their body. This tail is used for balance and communication, and Flufftails often hold it high and wave it as a sign of happiness or excitement.\n\nThese creatures are herbivorous and feed primarily on fruits, berries, and grasses. They are social animals and live in family groups made up of a mated pair of Flufftails and their offspring. They are very caring and protective of their young, who are born small and helpless.\n\nFlufftails are known for their curiosity and playfulness. They enjoy exploring their surroundings and playing with other members of their family. They have a complex system of vocalizations, including chirps, squeaks, and coos, that they use to communicate with one another.\n\nDespite their cute appearance, Flufftails are also skilled climbers and fast runners, able to escape any dangers that may be lurking in their environment. Overall, Flufftails are fascinating and delightful creatures that many would love to make their acquaintance."}]} +{"messages": [{"role": "user", "content": "Recite a nursery rhyme."}, {"role": "assistant", "content": "Twinkle, twinkle, little star, \nHow I wonder what you are! \nUp above the world so high, \nLike a diamond in the sky. \n\nWhen the blazing sun is gone, \nWhen he nothing shines upon, \nThen you show your little light, \nTwinkle, twinkle, all the night. \n\nThen the traveler in the dark \nThanks you for your tiny spark; \nHe could not see where to go, \nIf you did not twinkle so. \n\nIn the dark blue sky you keep, \nAnd often through my curtains peep, \nFor you never shut your eye \nTill the sun is in the sky. \n\nAs your bright but tiny spark \nLights the traveler in the dark, \nThough I know not what you are, \nTwinkle, twinkle, little star."}]} +{"messages": [{"role": "user", "content": "Describe a situation where a chatbot would be more useful than a human assistant."}, {"role": "assistant", "content": "A situation where a chatbot would be more useful than a human assistant is when handling multiple users simultaneously. For instance, suppose a company's customer support service receives thousands of inquiries daily, with many customers seeking to communicate with support representatives at the same time. In this scenario, a chatbot can handle multiple conversations simultaneously while providing instant and accurate responses to common inquiries, which would be impossible for a human assistant to achieve. Additionally, chatbots can operate 24/7 and do not need breaks or time off, meaning they are always available to handle customer inquiries. This capability minimizes wait times and improves the level of service that the company can provide to its customers."}]} +{"messages": [{"role": "user", "content": "Highlight any errors in this sentence.\nHe wrote a novel but it's was unsuccessful."}, {"role": "assistant", "content": "He wrote a novel but it's was unsuccessful.\n ^^^^ \nCorrection: He wrote a novel but it was unsuccessful."}]} +{"messages": [{"role": "user", "content": "Identify the sentence structure of the sentence, output the sentence structure type.\nAndrew gave money to the charity."}, {"role": "assistant", "content": "The sentence structure of \"Andrew gave money to the charity,\" is a simple sentence."}]} +{"messages": [{"role": "user", "content": "Describe the differences between e-commerce and conventional business."}, {"role": "assistant", "content": "E-commerce and conventional business are two different ways of conducting transactions and selling products or services to customers. Here are some key differences between them:\n\n- Mode of Transaction: The primary difference between e-commerce and conventional business lies in their mode of buying and selling. In E-commerce, transactions are done electronically on the internet, while in conventional business, transactions occur in person or face-to-face, where customers visit a physical store to purchase products.\n\n- Geographical Reach: E-commerce has a larger reach as compared to conventional businesses. As long as there is an internet connection, e-commerce businesses can sell products or services to anyone in any location. On the other hand, conventional businesses have a limited reach, as they rely on physical locations and local customers.\n\n- Inventory Management: Inventory management is also different in both models of businesses. In E-commerce, products can be listed without having them in stock, using dropshipping or just-in-time manufacturing methods. However, conventional businesses require physical products in stock in order to sell them.\n\n- Operational Cost: The operational cost for e-commerce businesses, including the cost of maintaining a website or online store, can be lower than conventional businesses, which have to pay for expenses such as rent, utilities, and other overhead costs associated with having a physical location.\n\n- Customer Interaction: In an e-commerce business, interaction with customers occurs mainly through digital channels like website chat, email, or social media. In contrast, a conventional business provides direct interaction with customers, in which the customers can touch, feel or even try on the products before purchasing them.\n\nOverall, while there are differences between E-commerce and conventional business, both have their unique advantages and disadvantages, and businesses often use a combination of both models to reach and serve their customer base."}]} +{"messages": [{"role": "user", "content": "List 5 questions for a job interview."}, {"role": "assistant", "content": "1. Can you tell us about your relevant experience for this role?\n2. How do you handle stress or pressure in the workplace?\n3. How do you approach problem-solving and decision-making in your work?\n4. Can you provide an example of a successful project you led or contributed to significantly?\n5. How do you go about continuing to develop your skills and knowledge in your field?"}]} +{"messages": [{"role": "user", "content": "Name 3 US States that have a desert climate."}, {"role": "assistant", "content": "Three US states that have a desert climate are Arizona, Nevada, and California."}]} +{"messages": [{"role": "user", "content": "Find all 3-digit palindromic numbers."}, {"role": "assistant", "content": "Palindromic numbers are numbers that remain the same when their digits are reversed. 3-digit palindromic numbers will have the same first and last digit. Here is a list of all the 3-digit palindromic numbers: \n\n101, 111, 121, 131, 141, 151, 161, 171, 181, 191, \n202, 212, 222, 232, 242, 252, 262, 272, 282, 292, \n303, 313, 323, 333, 343, 353, 363, 373, 383, 393, \n404, 414, 424, 434, 444, 454, 464, 474, 484, 494,\n505, 515, 525, 535, 545, 555, 565, 575, 585, 595, \n606, 616, 626, 636, 646, 656, 666, 676, 686, 696, \n707, 717, 727, 737, 747, 757, 767, 777, 787, 797, \n808, 818, 828, 838, 848, 858, 868, 878, 888, 898,\n909, 919, 929, 939, 949, 959, 969, 979, 989, 999."}]} +{"messages": [{"role": "user", "content": "Automatically generate a user account with a username and password"}, {"role": "assistant", "content": "I'm sorry, but as an AI Assistant, I am not able to automatically generate user accounts with usernames and passwords. However, I can guide you through the steps of creating one:\n\n1. Choose a platform or website where you would like to create an account.\n2. Click on the sign up or register button, usually found in the top right corner of the homepage.\n3. Enter your information, such as your name, email address, and birthdate. Some platforms may ask for additional information.\n4. Choose a unique and secure username. It is best to avoid common words and use a combination of letters and numbers.\n5. Create a strong password that is at least 8 characters long and includes a mix of upper and lowercase letters, numbers, and symbols.\n6. Confirm your password by typing it again.\n7. Follow any additional prompts or instructions to complete the account creation process.\n\nRemember to keep your account information secure and never share your password with anyone."}]} +{"messages": [{"role": "user", "content": "Tell me a story about a person who traveled worldwide."}, {"role": "assistant", "content": "Once upon a time, there was a young girl named Lily who dreamed of traveling to all the corners of the world. Growing up in a small village, she had always been curious about what lay beyond the mountains that surrounded her home. As soon as she was old enough, she packed her bags and set out on an adventure that would change her life forever.\n\nShe started her journey in Europe, where she explored the bustling streets of Paris, indulged in the finest Italian cuisine, and witnessed the majestic beauty of the Swiss Alps. From there, she journeyed to Africa, where she marveled at the vastness of the Sahara, walked with elephants in the grasslands, and danced with the locals in vibrant villages.\n\nAs she continued her travels, Lily made her way to Asia, where she explored towering temples in Thailand, hiked the Great Wall of China, and wandered the colorful markets of India. From there she sailed to South America, where she trekked through the Amazon rainforest, scaled the Andes Mountains, and swam with dolphins in the crystal clear waters of the Caribbean.\n\nWith each new destination, Lily learned more about the world and its incredible diversity. She discovered that despite our differences, we are all connected by the same hopes, dreams, and desires. And with that realization, she made it her mission to share her experiences and spread love and understanding wherever she went.\n\nYears passed, and Lily eventually returned home, but she was no longer the same curious girl who had set out on an adventure all those years ago. She was now a wise and worldly woman who had seen and experienced more than most could ever imagine. But she never forgot her journey and the lessons she learned, and she lived the rest of her life with a sense of wonder and gratitude for the incredible world that surrounded her."}]} +{"messages": [{"role": "user", "content": "Rate the difficulty of the following math problem from 1 to 5, 5 being the most difficult.\nSimplify: (4x + 8) / (2x - 4)"}, {"role": "assistant", "content": "2. The problem involves basic algebraic manipulation and simplification, making it a moderate difficulty problem."}]} +{"messages": [{"role": "user", "content": "Convert this equation into words.\ny = 2x^2 + 3x - 5"}, {"role": "assistant", "content": "The variable y is equal to two times the square of x, plus three times the value of x, minus five."}]} +{"messages": [{"role": "user", "content": "Give the temperature in Celsius when the temperature in Fahrenheit is 32."}, {"role": "assistant", "content": "32 degrees Fahrenheit is equivalent to 0 degrees Celsius."}]} +{"messages": [{"role": "user", "content": "Edit the following sentence for a more academic tone:\n\"Computers are great 'cause they can do so many things!\""}, {"role": "assistant", "content": "Computers are highly beneficial due to their capability to perform a vast array of functions."}]} +{"messages": [{"role": "user", "content": "Write a 200 word summary of the poem \"The Road Not Taken\""}, {"role": "assistant", "content": "“The Road Not Taken” is a famous narrative poem by Robert Frost, composed in 1916 and first published in 1920. The poem is told as first-person perspective where the speaker stands in front of a fork in the road in a yellow wood, pondering which path to take. One of the roads appears to be less traveled, with fewer signs of wear, but in hindsight, the speaker admits that the two paths were equally favored or less traveled. After a thoughtful pause, the speaker chooses the road less traveled, and the poem concludes with an ominous tone as the speaker reflects on this moment with a sigh, noting how choosing this path has made all the difference.\n\nThroughout the poem, Frost employs a variety of literary devices, including metaphor, imagery, and personification, to convey the poem's themes of choices and their consequences, the inevitability of change and uncertainty in life. The poem suggests that once a decision is made, it is impossible to go back and change what has happened or to truly know what would have happened if another path were chosen. It is a poem about the importance of free will, the complexities of human decision-making and the impact that even seemingly small choices can have on our futures."}]} +{"messages": [{"role": "user", "content": "Assemble the pieces of the sentence in the proper order.\nblack and - the - cat - white"}, {"role": "assistant", "content": "The cat is black and white."}]} +{"messages": [{"role": "user", "content": "Rearrange the criteria in the following statement in descending order by importance\naffordability, quality, usability"}, {"role": "assistant", "content": "1. Quality\n2. Usability\n3. Affordability"}]} +{"messages": [{"role": "user", "content": "Add a one line of dialogue that aptly sums up the situation.\nThe family had been planning their vacation for months, but as the date got closer the excitement started to dwindle."}, {"role": "assistant", "content": "\"Well, sometimes the anticipation of an event can be more thrilling than the event itself.\""}]} +{"messages": [{"role": "user", "content": "Suggest a way to organize one's email inbox."}, {"role": "assistant", "content": "One effective way to organize an email inbox is to use filters and labels to categorize emails. This method can help keep your inbox neat, clean and well-organized reducing the time spent for searching old emails. Here are the steps to follow:\n\n1. Identify common types of emails you receive regularly such as newsletters, work-related emails, personal emails, and promotional emails.\n\n2. Create labels for each category. Most email providers, like Gmail, have the option to create custom labels. For example, you might create labels such as \"Work,\" \"Finance,\" \"Personal,\" \"Newsletters,\" and \"Promotions.\"\n\n3. Set up filters. Filters automatically sort incoming emails into the appropriate label based on specific criteria. For example, you can set up a filter for emails from your boss to go directly into your \"Work\" label, or for emails from your bank to go into your \"Finance\" label.\n\n4. Utilize the \"star\" or \"important\" feature. If you have emails that require action or follow-up, you can mark them as important or star them so that you can easily find them later.\n\n5. Take advantage of the \"archive\" feature. Once you've read and dealt with an email, you can archive it to keep your inbox uncluttered. The email will still be accessible and searchable but won't be visible in your main inbox.\n\n6. Schedule regular clean-up time. Set aside a few minutes each day or once a week to go through your inbox, delete any unimportant emails, and move any that need to be filed into their appropriate labels.\n\nBy implementing these strategies, you can keep your email inbox organized, save time, and reduce stress."}]} +{"messages": [{"role": "user", "content": "Choose a movie and give 3 reasons why it is worth watching\nMovie: Inception"}, {"role": "assistant", "content": "Inception is an incredible movie that is definitely worth watching for several reasons.\n\n1. Unique and captivating storyline: Inception has an original and mind-bending plot that will keep you engaged from start to finish. The concept of entering someone's dreams to plant an idea in their mind is fresh, intriguing and full of possibilities, keeping you on the edge of your seat throughout the film.\n\n2. Visually stunning: The special effects in this film are outstanding. The dream worlds that are created are both breathtaking and surreal, and the action sequences are beautifully choreographed and executed. The visuals in Inception are a feast for the eyes and a testament to the creativity and skill of the filmmakers.\n\n3. Stellar cast and performances: Inception boasts an exceptional ensemble cast, including Leonardo DiCaprio, Joseph Gordon-Levitt, and Ellen Page, to name just a few. Each actor delivers a powerful and nuanced performance, bringing depth and emotion to their respective roles. The chemistry between the characters is fantastic, making for an engaging and moving cinematic experience."}]} +{"messages": [{"role": "user", "content": "Generate the first few lines of the poem."}, {"role": "assistant", "content": "In fields, where wildflowers softly sway,\nAnd sunlight streams a warm, golden ray,\nA gentle breeze rustles through the leaves,\nWhile nature hums and weaves its magic spell."}]} +{"messages": [{"role": "user", "content": "Develop a design for a logo representing a new company."}, {"role": "assistant", "content": "To design a logo for a new company, several factors should be taken into consideration. These include the company's products or services, target audience, values, and vision. With this information, here's a step-by-step process to develop the logo:\n\n1. Research the company: Obtain as much information about the company as possible. This should include its products or services, target audience, values, and vision. You can also check out what the competitors' logos look like to avoid looking similar to any.\n\n2. Create a mood board: Based on the gathered information, create a mood board of images, colors, typefaces, and other graphics that fit the brand's image.\n\n3. Sketch ideas: Begin brainstorming and sketching ideas based on the mood board. Don't be afraid to think outside the box.\n\n4. Refine concepts: Choose a few of the strongest logo concepts and refine them digitally.\n\n5. Choose typeface and colors: Choose a suitable typeface and color scheme that aligns with the brand's image. Use the mood board as a guide.\n\n6. Present the options: Share the logo options with the company and get feedback. If necessary, make further refinements.\n\n7. Finalize the design: Once the company has selected their preferred logo, finalize the design and prepare the necessary files for them to use.\n\nRemember to keep the logo simple, timeless, versatile, and memorable. The goal is to create a visual representation that captures the essence of the brand and can be easily recognized by its target audience."}]} +{"messages": [{"role": "user", "content": "Identify the five noteworthy aspects of the Declaration of Independence."}, {"role": "assistant", "content": "The Declaration of Independence is a historic document that announced the separation of the 13 North American British colonies from Great Britain. The five noteworthy aspects of the Declaration of Independence are as follows: \n\n1. It is a statement of equality - The Declaration of Independence contains the famous phrase, \"We hold these truths to be self-evident, that all men are created equal,\" which established the principle of equality of all people before the law. This was a radical notion at the time, as many societies still had systems of nobility and class hierarchy.\n\n2. It lists grievances against the British monarchy - The Declaration of Independence lists 27 specific grievances against King George III, accusing him of violating the rights of the colonists. These grievances included levying taxes without consent, dissolving colonial legislatures, and quartering troops in private homes.\n\n3. It asserts the right to self-government - The Declaration of Independence made it clear that the people of the colonies had the right to establish their own government, based on their own needs and desires. The document claimed that governments derive their powers \"from the consent of the governed,\" and that if a government fails to protect the rights of its citizens, the people have a right to alter or abolish it.\n\n4. It is a declaration of war - The Declaration of Independence was not simply a statement of principles; it was also a declaration of war. By announcing their separation from Great Britain, the colonies were defying the most powerful empire in the world, and effectively declaring war against it.\n\n5. It served as an inspiration for other nations - The Declaration of Independence has served as a model for other countries seeking independence, such as France and Haiti in the 18th century, and many Latin American countries in the 19th century. Its principles of liberty, equality, and self-government have influenced the development of democracy worldwide."}]} +{"messages": [{"role": "user", "content": "Translate the statement into a Boolean expression.\nAt least two of apples, oranges or bananas"}, {"role": "assistant", "content": "The statement \"At least two of apples, oranges, or bananas\" can be translated into the following Boolean expression: \n(apples AND oranges) OR (apples AND bananas) OR (oranges AND bananas)"}]} +{"messages": [{"role": "user", "content": "Describe the features of a new laptop model.\nThe new laptop model is equipped with a 8th Gen Intel i7 processor, 16GB of RAM, and an NVIDIA GeForce RTX 2060 GPU"}, {"role": "assistant", "content": "The new laptop model is the latest in computing technology and comes equipped with several cutting-edge features that enhance its performance and functionality. At its core is an 8th Gen Intel i7 processor, which provides the laptop with powerful and efficient processing capabilities. \n\nThe laptop has an impressive 16GB of RAM, which expands its multitasking capabilities and ensures it can run multiple applications with ease. This makes it perfect for both work and personal use, enabling the user to run multiple programs simultaneously without any slowdowns or glitches.\n\nOne of the standout features of this new laptop model is its NVIDIA GeForce RTX 2060 GPU. This advanced graphics card enables the laptop to handle complex graphical tasks, including gaming and video rendering, with ease. It also provides stunning visual clarity and realism, which makes using the laptop an immersive experience.\n\nIn summary, this new laptop model combines powerful hardware with advanced technology to deliver impressive performance and functionality. Its high-end processor, expansive RAM, and advanced graphics card make it a versatile and capable machine that can handle even the most demanding tasks."}]} +{"messages": [{"role": "user", "content": "predict labels for the following inputs.\nThe Eiffel Tower | Landmark\nThe Great Wall of China | Landmark"}, {"role": "assistant", "content": "It looks like both the Eiffel Tower and the Great Wall of China are classified as landmarks."}]} +{"messages": [{"role": "user", "content": "Develop a strategy for solving a complex problem.\nBookshelves are full and disorganized."}, {"role": "assistant", "content": "One approach to solving the complex problem of full and disorganized bookshelves could be to follow these steps:\n\n1. Define the problem: In this case, the problem is that the bookshelves are cluttered and arranging them is becoming difficult.\n\n2. Evaluate the current state: Assess the extent of the disorganization and clutter on the bookshelves, and categorize the books according to their size, genre, author, or any other preferred criteria.\n\n3. Devise a plan: Consider different organizing methods, such as alphabetical order, color-coordination, or grouping by genre. Decide on the most effective and aesthetically pleasing method for arranging the books.\n\n4. Take action: Start by taking all the books off the shelves and sorting them according to the chosen criteria. Dust the shelves and clean them up before putting the books back in their designated order. Consider donating or selling any books that are no longer needed or wanted to free up space on the shelves.\n\n5. Maintain the order: In order to keep the bookshelves organized, make a conscious effort to stick to the chosen method of arrangement. When new books are added, ensure that they are put in the right place according to the set criteria.\n\n6. Evaluate the results: After implementing the plan, evaluate the results and see if the bookshelves are now more organized and easy to navigate. If necessary, adjust the arrangement method to better suit the needs and preferences.\n\nIn conclusion, solving the problem of disorganized bookshelves requires assessing the current state, devising and implementing a plan, maintaining the order, and evaluating the results for further improvement."}]} +{"messages": [{"role": "user", "content": "Describe your dream house."}, {"role": "assistant", "content": "As an AI assistant, I do not possess the ability to dream or desire for material possessions such as a house. However, based on common human desires for a dream house, here is a description that may fit the bill. \n\nThe dream house is a breathtaking, spacious home nestled in the heart of the countryside, surrounded by lush greenery and scenic views. The architecture is a blend of classic and modern design, with large windows that allow natural light to flood in, and the rooms are spacious, with high ceilings and an open floor plan. \n\nThe house boasts a spacious kitchen with state-of-the-art appliances, granite countertops, and an island that doubles as a breakfast bar. The living room is warm and inviting, with a fireplace for cold nights, and large doors leading out to a wrap-around porch. \n\nThe bedrooms are grand, each with their own en suite bathroom, and the master suite is particularly luxurious, with a walk-in closet, a spa-like bathroom, and a balcony overlooking the picturesque scenery. \n\nThe house also features its own gym, home theater, and game room. The outdoor space is just as grand, with an elegant pool area, manicured gardens, and plenty of space for outdoor activities.\n\nAll in all, this dream house is a luxurious, spacious, and elegant home, built for comfort and relaxation, and designed to be a peaceful retreat from the hustle and bustle of everyday life."}]} +{"messages": [{"role": "user", "content": "Create a function to sort a list from highest to lowest.\n[10, 30, 20, 50, 40]"}, {"role": "assistant", "content": "Here is a function that will sort a list from highest to lowest:\n\n```python\ndef sort_list_high_to_low(lst):\n lst.sort(reverse=True)\n return lst\n\n# example\nnumbers = [10, 30, 20, 50, 40]\nsorted_numbers = sort_list_high_to_low(numbers)\nprint(sorted_numbers)\n```\n\nThe function `sort_list_high_to_low` takes a list as an input and sorts it in descending order using the `sort()` method by setting its `reverse` parameter to `True`. The sorted list is then returned as the output.\n\nIn this example, the given input list `[10, 30, 20, 50, 40]` is passed to the function and the output is the sorted list `[50, 40, 30, 20, 10]`."}]} +{"messages": [{"role": "user", "content": "Rewrite the following sentence to include two adjectives.\nThe car is fast."}, {"role": "assistant", "content": "The sleek, red car is incredibly fast."}]} +{"messages": [{"role": "user", "content": "Generate a headline for an article discussing the importance of backup systems."}, {"role": "assistant", "content": "\"Don't Risk Losing it All: The Critical Importance of Backup Systems\""}]} +{"messages": [{"role": "user", "content": "Spell out this acronym\nCAPTCHA"}, {"role": "assistant", "content": "CAPTCHA stands for Completely Automated Public Turing test to tell Computers and Humans Apart."}]} +{"messages": [{"role": "user", "content": "Create a list of 5 synonyms for the word 'persuade'."}, {"role": "assistant", "content": "1. Convince\n2. Influence\n3. Coax\n4. Motivate\n5. Prompt"}]} +{"messages": [{"role": "user", "content": "Create a program that takes an array and outputs the largest element\n[1, 3, 15, 20, 8, 9]"}, {"role": "assistant", "content": "Here is one way you could write this program in Python:\n\n```python\ndef largest_element(array):\n max = array[0]\n for num in array:\n if num > max:\n max = num\n return max\n\narray = [1, 3, 15, 20, 8, 9]\nresult = largest_element(array)\nprint(result)\n```\n\nIn this code, we define a function, `largest_element`, which takes an array as an input, and returns the largest element of that array. \nWithin the function, we initialize a variable `max` to the first element of the array. Then we iterate through the array using a for loop and compare the current element with the `max` element. If the current element is greater than `max`, then we update `max` with that value. At the end of the loop, we return the final value of `max`, which is the largest element in the array.\nWe then call the function with the provided array as input, store the result in a variable, and print it. \n\nFor the provided input of [1, 3, 15, 20, 8, 9], the program would output: `20`"}]} +{"messages": [{"role": "user", "content": "Create a book cover for the book \"Call Me by Your Name\"\nNo input"}, {"role": "assistant", "content": "I'm sorry, but I'm an AI conversationalist and unable to create images or book covers. I recommend working with a graphic designer or using design software to create a visual representation of the book title \"Call Me by Your Name.\""}]} +{"messages": [{"role": "user", "content": "Answer this analogy: Teacher is to Student as Leader is to"}, {"role": "assistant", "content": "Follower."}]} +{"messages": [{"role": "user", "content": "The following sentence is incomplete. Please complete it.\nHumans have an innate desire to"}, {"role": "assistant", "content": "Humans have an innate desire to connect with others and form meaningful relationships."}]} +{"messages": [{"role": "user", "content": "What advice would you give to a child just starting school?"}, {"role": "assistant", "content": "Starting school can be both an exciting and nervous time, but don't worry, here are some helpful tips to make this transition easier:\n\n1. Be prepared: Make sure you have all the supplies you need and that you are ready for your first day.\n\n2. Get a good night's sleep: It's important to get enough sleep so you can feel rested and ready to learn.\n\n3. Eat breakfast: A good breakfast will give you the energy you need to be alert and focused in the classroom.\n\n4. Listen and follow directions: Make sure to pay attention to your teacher and the instructions they give you. This will help you to learn and do your best in class.\n\n5. Ask for help if you need it: If you don't understand something, don't be afraid to ask your teacher for help.\n\n6. Make friends: School is a great place to meet new people and make new friends. Don't be shy, try to say hello to people and get to know them.\n\n7. Be respectful: Treat your classmates, teachers, and school staff with respect and kindness. This will help create a positive environment in the classroom.\n\nRemember, everyone gets nervous at the beginning but you'll get the hang of it in no time. Just do your best and have fun, school is a great place to learn and grow!"}]} +{"messages": [{"role": "user", "content": "Create an interactive quiz with at least three questions"}, {"role": "assistant", "content": "Sure, let me create an interactive quiz for you. Here are three questions:\n\n1. What is the capital of France?\na) New York\nb) Berlin\nc) Paris\nd) London\n\n2. Which planet in our solar system is known as the Red Planet?\na) Earth\nb) Mars\nc) Jupiter\nd) Venus\n\n3. Who invented the light bulb?\na) Thomas Edison\nb) Alexander Graham Bell\nc) Isaac Newton\nd) Albert Einstein \n\nSo, are you ready to start the quiz? Let me know which question you would like to answer first."}]} +{"messages": [{"role": "user", "content": "Deleted the second-to-last sentence of this paragraph.\nI wanted to take a break from all the bad news in the world, so I decided to get away for a bit. I went to the beach and enjoyed the sun, sand and waves. I met some new people and had some great conversations. It was a great time to get away from the worries of the world."}, {"role": "assistant", "content": "I wanted to take a break from all the bad news in the world, so I decided to get away for a bit. I went to the beach and enjoyed the sun, sand and waves. It was a great time to get away from the worries of the world."}]} +{"messages": [{"role": "user", "content": "Come up with a unique story idea involving a magic wand"}, {"role": "assistant", "content": "In a world of magic and enchantment, there was a wand maker who was renowned for crafting the most powerful wands. Every wizard and witch aspired to acquire one of his creations, and no one ever left his shop disappointed. However, the wand maker had a secret; he had a special wand that he kept hidden in his shop, away from prying eyes. This wand had a remarkable ability; it could bring the wielder's deepest desires to life.\n\nOne day, the wand maker's shop was burgled, and the special wand was stolen. It fell into the hands of a young orphan who had no idea of its powers. All the orphan wanted was a family and a place to call home. Without realizing it, the orphan waved the wand, and it worked its magic. A portal opened, and the orphan was transported to a parallel world where his parents were alive, and he had siblings and a loving home.\n\nThe orphan soon discovered that in this world, he was the chosen one, destined to defeat an evil overlord who was terrorizing the land. The wand became his weapon and his guide, leading him on a quest to save his new home and the family he had always longed for.\n\nBut as the orphan's powers grew, so did the dangers that surrounded him. The wand's magic was unpredictable, and the orphan had to learn to control his deepest desires to prevent the wand from causing havoc. The ultimate test came when the orphan was faced with a choice between saving his family and defeating the overlord. The wand's power and the orphan's courage were put to the ultimate test in an epic battle that would determine the fate of the world."}]} +{"messages": [{"role": "user", "content": "What is the scientific name for the Antarctic blue whale?"}, {"role": "assistant", "content": "The scientific name for the Antarctic blue whale is Balaenoptera musculus intermedia."}]} +{"messages": [{"role": "user", "content": "Evaluate the opinion expressed in the given sentence.\nClimate change is not a real issue."}, {"role": "assistant", "content": "The sentence expresses the opinion that climate change is not a genuine or pressing concern. This view is not supported by the vast majority of scientific evidence, which overwhelmingly indicates that climate change is a real and significant issue that requires urgent action."}]} +{"messages": [{"role": "user", "content": "Identify the parts of speech for the following sentence and provide corresponding tags.\nThe dog chased the cat."}, {"role": "assistant", "content": "The - Determiner (DT)\ndog - Noun (NN)\nchased - Verb (VBD)\nthe - Determiner (DT)\ncat - Noun (NN)\n\nComplete sentence with corresponding tags: The/DT dog/NN chased/VBD the/DT cat/NN."}]} +{"messages": [{"role": "user", "content": "Write a jargon-free description of the given concept.\nConcept: Cloud Computing"}, {"role": "assistant", "content": "Cloud computing is a way to access and store data, programs, and applications over the internet instead of on a computer's hard drive or a local server. This means that you can use the services and resources of a network of remote servers to store, manage and process information, rather than relying on a single local computer or server. This provides users with a convenient and cost-effective way to access and use the computing resources they need, whenever they need them, without having to worry about managing or maintaining the underlying infrastructure."}]} +{"messages": [{"role": "user", "content": "Give an example of self-care activity."}, {"role": "assistant", "content": "One example of a self-care activity is taking a relaxing bath. This involves filling a bathtub with warm water, adding bubbles or bath salts, lighting some candles, and sinking into the warm water to unwind and de-stress. This simple activity can help to soothe tense muscles, calm the mind, and reduce feelings of anxiety and stress."}]} +{"messages": [{"role": "user", "content": "Create a character description of Anna from Frozen"}, {"role": "assistant", "content": "Anna is the lively and effervescent Princess of Arendelle and the younger sister of Elsa. She is optimistic, caring, and always willing to help those in need. She has a round face, rosy cheeks, and sparkling blue eyes. Her auburn hair is styled in two braids and her infectious smile lights up the room. Anna is brave, determined, and has an unbreakable bond with her sister, Elsa. She is often seen wearing a dress with a black bodice, light blue skirt, and a magenta cape. Anna is known for her adventurous spirit and unwavering love for her family and friends. She is a beloved Disney character and a true hero in the story of Frozen."}]} +{"messages": [{"role": "user", "content": "Write a blog post about a theme mentioned in the text.\nThe story is about a world without technology and the impact it had on the people."}, {"role": "assistant", "content": "The Impact of a World Without Technology\n\nImagine living in a world without technology. No smartphones, no internet, no computers, no cars, and no electricity. How would life be without the convenience and the speed that technology provides us today? This scenario might sound like a scene from a fictional story, but the reality is, technology has come a long way and has become an integral part of our daily lives.\n\nTechnology has made many aspects of our lives easier, from simplifying daily tasks to providing instant access to information and entertainment. However, it's worth considering the impact it has on our lives and the world around us.\n\nIn a world without technology, we would be forced to slow down, to physically interact with the world around us, rather than relying on the virtual world. It would likely lead to a more active lifestyle as we would rely more on physical activities in our daily routines without the distractions of screens and devices. The absence of technology would also mean that we would have to rely more on personal interactions and face-to-face communication, which could lead to stronger connections between people and a greater appreciation for those around us.\n\nWithout technology, we would also have to learn to live without the many conveniences it provides. Simple tasks like cooking, washing clothes, and traveling long distances would become more laborious and time-consuming. On the other hand, it could lead to a simpler, more sustainable way of living, as we would be forced to rely on natural resources and live in harmony with nature.\n\nA world without technology may seem like a daunting prospect, but it would also have many positive impacts on our lives and the world around us. We would learn to appreciate the simple pleasures in life and engage with the world around us on a deeper, more meaningful level. That being said, technology is here to stay, and it is up to us to use it responsibly and to balance its benefits with its impact on our lives and the world around us."}]} +{"messages": [{"role": "user", "content": "Select the best website font from the 3 options given and provide a brief explanation for your choice.\nFont options - Montserrat, Roboto, Alegreya"}, {"role": "assistant", "content": "Out of the given font options - Montserrat, Roboto, and Alegreya, the best website font would be Roboto. This is because Roboto is a clean and modern sans-serif font that has a friendly and professional appearance. It effectively balances between letterform and geometric shapes and has excellent legibility and readability in a variety of sizes, making it suitable for displaying text on screens. Furthermore, it is versatile and can be used for both headers and body text, and its wide range of weights and styles make it adaptable for different design purposes."}]} +{"messages": [{"role": "user", "content": "Why should you learn Python?"}, {"role": "assistant", "content": "There are several reasons why learning Python can be beneficial:\n\n1) Versatility: Python is a versatile and powerful programming language that has a wide range of applications. It can be used for web development, data analysis, machine learning, artificial intelligence, and scientific computing, among others.\n\n2) Easy to learn: Python has a simple and readable syntax that makes it one of the most beginner-friendly programming languages. Its straightforward nature makes coding in Python more manageable and efficient, even for those with no programming background.\n\n3) High demand: Python is one of the most in-demand programming languages in the job market. With its wide range of applications, Python developers are highly sought after, and there is no shortage of employment opportunities for those with Python skills.\n\n4) Large Community: One of the greatest strengths of Python is its large and supportive community. Python has a massive presence online, with numerous tutorials, forums, and support groups. This makes it easy to find support and assistance when you encounter difficulties while learning or working with the language.\n\n5) Open-Source: Python is an open-source language, meaning that it is freely available to the public. This allows developers to contribute to its growth and development, and it means that there is a wealth of free resources available to help you learn and use the language."}]} +{"messages": [{"role": "user", "content": "Write a conclusion for the following movie review: \"The Green Mile was a captivating and moving movie.\""}, {"role": "assistant", "content": "In conclusion, The Green Mile is a highly recommended film that successfully captures the heart of its audience. With its powerful storytelling and emotional depth, this movie is sure to leave a lasting impact on all who watch it."}]} +{"messages": [{"role": "user", "content": "Given a description of the scenario, describe how the character feels.\nA young girl is entering an unfamiliar room"}, {"role": "assistant", "content": "The character, in this case the young girl, might feel nervous, hesitant or uncomfortable as she enters a room that is unfamiliar to her. She may also be curious or inquisitive to explore and discover the new environment."}]} +{"messages": [{"role": "user", "content": "Determine whether the car is moving or not.\nThe car engine is roaring and the car is making a rapid motion."}, {"role": "assistant", "content": "Based on the input, it appears that the car is moving."}]} +{"messages": [{"role": "user", "content": "Write a short persuasive passage about the effects of climate change"}, {"role": "assistant", "content": "Climate change is a growing concern that affects us all. Its far-reaching effects are evident in the melting of polar ice caps, rising sea levels, and the increases in frequency and severity of natural disasters, such as hurricanes and wildfires. Moreover, it's causing drastic shifts in global climate patterns leading to prolonged droughts or intense rainfall, and creating health and ecological issues. The root cause of climate change is human activity, particularly our reliance on fossil fuels, deforestation, and unsustainable agricultural practices. Inaction is no longer an option. We must act now by reducing our carbon footprint, transitioning to renewable energy, preserving our natural ecosystems, and implementing sustainable practices in our daily lives. The consequences of inaction are dire and permanent, and the time to act is now. By addressing climate change today, we can build a cleaner, healthier, and more sustainable future for generations to come."}]} +{"messages": [{"role": "user", "content": "Create a new sequence out of the given numbers.\nNumbers: 0.2, 3, 5, 6"}, {"role": "assistant", "content": "[0.2, 3, 5, 6]\n\nThere are several ways to create a sequence out of the given numbers. Here's one possibility:\n\nSequence: [0.2, 3.2, 6.2, 9.2]"}]} +{"messages": [{"role": "user", "content": "Find the surface area of a cube with sides 3 cm long.\n3 cm"}, {"role": "assistant", "content": "The surface area of a cube is given by the formula A=6a^2, where A is the surface area and a is the length of each side. So, for a cube with sides 3 cm long, the surface area would be A = 6 x 3^2 = 6 x 9 = 54 cm^2."}]} +{"messages": [{"role": "user", "content": "What was the main takeaway of the article?\nThe article, \"The State of the Climate in 2020\" discussed the effects of climate change on the environment and its impact on global temperatures, sea levels, and other elements of the planet's ecology."}, {"role": "assistant", "content": "The main takeaway of the article, \"The State of the Climate in 2020,\" is that climate change is having significant effects on the environment, with impacts on global temperatures, sea levels, and other elements of the planet's ecology."}]} +{"messages": [{"role": "user", "content": "Create a joke with the punchline \"That's why they call it a holdup\"."}, {"role": "assistant", "content": "Why did the bank robber take a nap on the job? Because he wanted to make sure he pulled off the crime without any slip-ups. And that's why they call it a holdup!"}]} +{"messages": [{"role": "user", "content": "Suggest an activity to learn more about the animal biodiversity."}, {"role": "assistant", "content": "One activity that could help someone learn more about animal biodiversity is to visit a local zoo or wildlife sanctuary. These facilities offer an opportunity to observe and learn about a wide variety of animals from different regions of the world. You can also participate in educational tours, talks, and interactive demonstrations which provide more in-depth information about the different species and their habitats. Additionally, many zoos and sanctuaries actively participate in conservation efforts to protect and preserve biodiversity, so it's a great chance to learn about the role of these organizations in protecting our planet's animal life."}]} +{"messages": [{"role": "user", "content": "Can you explain how blockchains work?"}, {"role": "assistant", "content": "A blockchain is a digital ledger of transactions, which is duplicated and distributed across a network of computer systems. It is a decentralized and distributed database that is managed by different participants.\n\nEach block in the blockchain contains several transactions and when new transactions occur, they are broadcast to every participant in the network. After verification, the transaction is approved and a new block is formed containing the new transaction data. This new block is then added to the existing blockchain and shared with every participant in the network.\n\nSince the blockchain is distributed and decentralized, there is no central authority that manages or verifies the transactions. Instead, this is done by consensus and through a process called mining. Mining involves solving a complex mathematical problem and adding a verified block to the blockchain, and as a reward, new bitcoins are generated and awarded to the miner.\n\nThe data in the blocks is secured using cryptography, where the data is converted into a fixed size of alphanumeric codes through a cryptographic hash function. Each block contains the hash of the previous block, so changing any data in a block would change the hash of that block and all the subsequent blocks, making it very difficult to tamper with the data.\n\nOverall, blockchains provide a secure, transparent, and decentralized way for recording and verifying transactions, making it a popular technology for applications such as cryptocurrencies, supply chain management, and voting systems."}]} +{"messages": [{"role": "user", "content": "Write a code in Python that will generate a random string of 5 upper-case letters."}, {"role": "assistant", "content": "Sure. Here's the code in Python 3 that generates a random string of 5 upper-case letters:\n\n```python\n\nimport string\nimport random\n\ndef random_string(length: int = 5):\n result = ''.join(random.choice(string.ascii_uppercase) for _ in range(length))\n return result\n \n# Example\nprint(random_string(5))\n```\n\nThis function generates a random string with a default length of 5 using the `random.choice()` method to pick a random character from a set of upper-case ASCII characters defined in `string.ascii_uppercase`. The length parameter is optional and can be adjusted to generate a string of any length."}]} +{"messages": [{"role": "user", "content": "Generate a recipe for black bean tacos"}, {"role": "assistant", "content": "Ingredients:\n- 1 can black beans, rinsed and drained\n- 1/2 onion, chopped\n- 2 cloves of garlic, minced\n- 1 tbsp olive oil\n- 1 tsp chili powder\n- 1/2 tsp cumin\n- Salt and Pepper to taste\n- 6 small tortillas\n- Freshly chopped cilantro\n- Shredded cheese\n- Your favorite taco toppings (diced tomatoes, diced avocado, salsa, lettuce)\n\nInstructions:\n\n1. Heat the olive oil in a pan over medium heat. Add the chopped onion and minced garlic, and sauté until the onions are softened and translucent, for about 3 minutes.\n\n2. Add the drained black beans, chili powder, and cumin to the pan. Season with salt and pepper to taste. Cook for another 5 minutes, stirring occasionally, until beans are heated through.\n\n3. While the beans are cooking, heat the tortillas in the microwave or on the stove-top, according to the package instructions.\n\n4. To assemble the tacos, layer the bean mixture on the tortillas, and top with the shredded cheese, cilantro, and any other desired toppings.\n\n5. Serve immediately and enjoy your delicious black bean tacos!"}]} +{"messages": [{"role": "user", "content": "Create a spreadsheet to track monthly expenses"}, {"role": "assistant", "content": "Here is a template for a basic monthly expenses spreadsheet:\n\n| Category | Expense | Amount | Date | Notes |\n| --- | --- | --- | --- | --- |\n| Housing | Rent/Mortgage | 1200.00| 4/1/2022 | |\n| Utilities | Electricity | 100.00| 4/5/2022 | |\n| Utilities | Water | 60.00| 4/5/2022 | |\n| Food | Groceries | 400.00| 4/15/2022 | |\n| Transportation | Gas | 100.00| 4/20/2022 | |\n| Health | Doctor's Appointment | 20.00| 4/30/2022 | Copay |\n| Total | | 1880.00 | | |\n\nTo use this template, you can input your own expenses and amounts in the relevant columns. The categories and expenses can be customized to fit your personal needs. The date column allows you to track when each expense is due or was paid. The notes column can be used for any additional details or comments. At the bottom of the Amount column, the total expenses for the month can be calculated."}]} +{"messages": [{"role": "user", "content": "Come up with three adjectives to describe the color red."}, {"role": "assistant", "content": "Vibrant, bold, and fiery."}]} +{"messages": [{"role": "user", "content": "Brainstorm a way to reduce deforestation."}, {"role": "assistant", "content": "One way to reduce deforestation is to implement sustainable forestry practices. This can include measures such as selective cutting (harvesting only mature trees and leaving younger ones intact), maintaining buffer zones around bodies of water, and reducing clearcutting (where all trees in a given area are removed) in favor of more selective methods. Additionally, enforcing strict regulatory measures and providing economic incentives to companies that adhere to sustainable practices can encourage responsible and sustainable forest management. Another approach is to promote alternatives to deforestation-driven industries, such as agriculture, by expanding opportunities for sustainable income generation, such as ecotourism, agroforestry, and conservation-related jobs. Finally, increasing public awareness and promoting conservation education can help foster a culture of responsible land use and build support for sustainable forestry practices."}]} +{"messages": [{"role": "user", "content": "Create a function which generates all pairs of numbers within a given range with a given sum.\nRange: 0 - 10\nSum: 10"}, {"role": "assistant", "content": "Here is a function in Python that generates all pairs of numbers within a given range with a given sum:\n\n```python\ndef generate_pairs_in_range_with_sum(range_start, range_end, desired_sum):\n pairs = []\n for i in range(range_start, range_end + 1):\n for j in range(range_start, range_end + 1):\n if i + j == desired_sum:\n pairs.append((i, j))\n return pairs\n\nrange_start = 0\nrange_end = 10\ndesired_sum = 10\npairs = generate_pairs_in_range_with_sum(range_start, range_end, desired_sum)\n\nprint(pairs)\n```\n\nThis will output the following list of pairs when range is from 0 to 10 and desired sum is 10:\n\n`[(0, 10), (1, 9), (2, 8), (3, 7), (4, 6), (5, 5), (6, 4), (7, 3), (8, 2), (9, 1), (10, 0)]`\n\nThe function iterates through each number within the given range using two nested for loops. For each iteration, it checks if the sum of the two loop variables is equal to the desired sum. If it is, it adds the pair of numbers to a list and returns the list after all iterations are complete."}]} +{"messages": [{"role": "user", "content": "Given a list of numbers, add them up and provide the result\n2, 4, 7"}, {"role": "assistant", "content": "The result of adding up the numbers 2, 4, and 7 is 13."}]} +{"messages": [{"role": "user", "content": "Re-write the given sentence in the imperative form.\nIt's important to protect the environment."}, {"role": "assistant", "content": "Protect the environment."}]} +{"messages": [{"role": "user", "content": "Determine a network architecture that can identify emotion in text."}, {"role": "assistant", "content": "To identify emotion in text, we can use a deep learning model designed specifically for natural language processing (NLP) tasks. Here is an example of a network architecture that can be used for this purpose:\n\n1. Input Layer: The input to the model would be a sequence of text in the form of word embeddings, which represent words as vectors of fixed size. This helps the model understand the semantic meaning of words and the relationships between them.\n\n2. Recurrent Layer: The first hidden layer can be a recurrent neural network (RNN) or its variants like LSTM (Long Short-Term Memory) or GRU (Gated Recurrent Unit). These layers are designed to capture the contextual information of the input text by keeping track of the previous hidden states in the sequence.\n\n3. Attention Layer: An optional attention layer can be added after the recurrent layer to help the model focus on the most important parts of the input sequence while making predictions.\n\n4. Fully Connected Layer: The output from the recurrent or attention layer can be fed to a fully connected neural network with one or more hidden layers. These layers can learn the non-linear relationships between the input features and the target labels.\n\n5. Output Layer: The final output layer should have the same number of units as the number of emotions being classified. A softmax activation function can be applied to the output to generate probabilities for each emotion class.\n\nThis is just one example of a network architecture that can be used to identify emotions in text. Other options include using pre-trained language models like BERT or transformers as the base of the model and fine-tuning them on the specific emotion classification task. The choice of architecture would depend on the nature of the text data being used and the specific emotions being classified."}]} +{"messages": [{"role": "user", "content": "What is the average adult height in centimeters?"}, {"role": "assistant", "content": "The average height for adult humans varies widely between individuals and populations, but the global average height for adult men is about 171 cm (5'7\"), while for adult women it is about 159 cm (5'3\"). However, these numbers also change based on factors such as age, nutrition, health, and genetic background."}]} +{"messages": [{"role": "user", "content": "Write a single sentence summarizing the given excerpt\nThe revolution began in 1789 when the people of France demanded the deposition of King Louis XVI."}, {"role": "assistant", "content": "The French Revolution started in 1789 with the public's call for the removal of King Louis XVI from power."}]} +{"messages": [{"role": "user", "content": "Come up with three plausible strategies to diversify a product offer."}, {"role": "assistant", "content": "1. Conduct market research and analyze customer preferences: One effective way to diversify a product offer is to conduct in-depth market research and analyze the preferences of the target customer group. This could involve collecting data on trends, studying feedback from existing customers, and conducting surveys to find out what they want that your company is not offering. Based on the insights drawn from this research, a company can identify gaps in the market and opportunities to expand its product range to meet the changing needs and preferences of its customer base.\n\n2. Expand into new markets or demographic segments: Another strategy to diversify a product offer is to expand into new markets or target different demographic groups. This could mean introducing products tailored to the needs and preferences of different age, gender, or socio-economic groups, or expanding into new geographic regions, both domestically and abroad. By doing so, a company can broaden its customer base, reduce its dependence on any one market, and increase its potential for revenue growth.\n\n3. Collaborate with other businesses and engage in co-branding: A company can also diversify its product offer by collaborating with other businesses and engaging in co-branding initiatives. By doing so, it can leverage the unique strengths and capabilities of its partner companies to create innovative new product offerings that appeal to a broader customer base. For example, a fashion brand could collaborate with a technology company to develop a line of smart clothing or accessories, thereby expanding its product range beyond traditional fashion items."}]} +{"messages": [{"role": "user", "content": "List two advantages of using a content management system (CMS)."}, {"role": "assistant", "content": "1. Simplified Content Management: One of the major advantages of using a CMS is that it simplifies the process of creating, organizing, editing and publishing content on a website. A CMS comes with an intuitive and user-friendly interface that allows users to manage content easily without having prior knowledge of programming or web development. This means that even non-technical users can update and maintain the website content with relative ease.\n\n2. Customization and Flexibility: With a CMS, it’s easy to customize the look and feel of a website using pre-built templates and themes, and to add new functionalities through the use of plugins, modules, or extensions. This level of flexibility allows users to build and manage dynamic websites that can be tailored to their specific needs and adjusted as the business grows or as user demands change."}]} +{"messages": [{"role": "user", "content": "Identify the name of the city in this scene.\nThe sun was setting over the ancient city, its golden rays shining on the minarets of its many mosques."}, {"role": "assistant", "content": "The name of the city is not explicitly stated in the given scene."}]} +{"messages": [{"role": "user", "content": "Rewrite this sentence using another grammatical structure.\nI will be helping her."}, {"role": "assistant", "content": "She will be receiving help from me."}]} +{"messages": [{"role": "user", "content": "Generate a simple task-oriented conversation up to 5 turns."}, {"role": "assistant", "content": "User: Hey AI, can you help me set my alarm for tomorrow morning at 7am?\nAI: Yes, of course. I've set your alarm for 7am tomorrow morning. Is there anything else you need?\nUser: Yes, can you remind me to start preparing breakfast when the alarm goes off?\nAI: Sure. I’ve set a reminder for you to start preparing breakfast at 7am when your alarm goes off. \nUser: Great! Can you also add eggs, bread and orange juice to my grocery list.\nAI: I have added eggs, bread, and orange juice to your grocery list. Do you need anything else?\nUser: No, that’s everything for now. Thanks a lot!\nAI: You're welcome. Have a great day!"}]} +{"messages": [{"role": "user", "content": "Create a movie title using one of the given words.\nCloud"}, {"role": "assistant", "content": "Cloud in The Sky: A Love Story"}]} +{"messages": [{"role": "user", "content": "Design a machine learning algorithm for predicting the price of a house."}, {"role": "assistant", "content": "The task of predicting the price of a house can be approached through a supervised machine learning algorithm. Here is a high-level overview of how a machine learning algorithm that predicts the price of a house can be designed:\n\n1. Data collection and preprocessing: Collect a large dataset of information on houses and their prices. This dataset should be cleaned and preprocessed, which includes removing any incomplete or irrelevant data points, handling missing values, and standardizing the data.\n\n2. Feature selection: Identify the features or variables in the dataset that are most relevant to predicting the price of a house. These could include the location, size, number of bedrooms and bathrooms, and age of the house, among others.\n\n3. Model selection: Choose an appropriate machine learning model for the task. Some common models used for regression tasks, such as predicting a continuous value like the price of a house, include linear regression, decision trees, and neural networks.\n\n4. Training and validation: Divide the dataset into a training and validation set. Train the chosen model on the training set and assess its performance on the validation set. If the model is not performing well, adjust its parameters or try a different model.\n\n5. Evaluation: Once a satisfactory model is chosen, it can be evaluated on a test set, which should be separate from the training and validation sets, to assess its real-world performance.\n\n6. Deployment: Once the model has been trained and evaluated, it can be deployed in a production environment to make predictions on new data points.\n\nThis is just a high-level overview of how a machine learning algorithm for predicting the price of a house can be designed. The specific details of the algorithm, such as the type of model used and the features selected, may vary depending on the specifics of the dataset and the problem at hand."}]} +{"messages": [{"role": "user", "content": "Make a list of five documents a student will need for a college application."}, {"role": "assistant", "content": "Here are five important documents a student will typically need when applying to college: \n\n1. Transcript: Colleges will need to see the student's academic record in the form of an official transcript, which typically includes the student's grades and courses taken throughout high school. \n\n2. Test scores: Many colleges require standardized test scores, such as the SAT or ACT, as part of the application process. The student will need to submit these scores as part of their application. \n\n3. Letters of recommendation: Many colleges request letters of recommendation from teachers, counselors, or other adults who know the student well and can speak to their strengths, achievements, and potential. \n\n4. Personal essay: Many college applications require a personal essay, which gives the student a chance to share their own story, goals, and motivations for attending college. \n\n5. Application form: The student will need to fill out and submit the college's application form, which will include basic information about the student, their academic record, and any extracurricular activities or achievements."}]} +{"messages": [{"role": "user", "content": "Name three advantages of using the given product.\nInternet of Things"}, {"role": "assistant", "content": "1. Convenience and Efficiency: Internet of Things (IoT) devices can automate many tasks in our daily lives, making them more convenient and efficient. For example, with smart home technology, users can control lighting, temperature, and security systems remotely using their smartphones.\n\n2. Data Collection and Analysis: IoT devices are equipped with sensors that collect data and share it to the cloud for analysis. This enables users to keep track of usage patterns, identify trends and make better decisions based on the data collected.\n\n3. Cost Savings: By automating tasks, reducing energy consumption, and optimizing resource usage, IoT devices can help users save money. For example, a smart thermostat can learn the user’s schedule and adjust the temperatures accordingly, leading to savings on energy bills."}]} +{"messages": [{"role": "user", "content": "Construct a function to classify words into nouns, verbs, and adjectives."}, {"role": "assistant", "content": "One way to classify words into their part of speech is to use natural language processing (NLP) techniques such as Parts of Speech (POS) tagging. Here, I will construct a function using Python's Natural Language Toolkit (NLTK) library that classifies words into nouns, verbs, and adjectives:\n\n```python\n\nimport nltk\nfrom nltk.corpus import wordnet\n\nnltk.download('averaged_perceptron_tagger')\nnltk.download('wordnet')\n\ndef classify_words(word):\n # use the nltk library for POS tagging\n part_of_speech = nltk.pos_tag([word])[0][1]\n \n # define the POS tags that represent nouns, verbs, and adjectives\n noun_tags = [\"NN\", \"NNS\", \"NNP\", \"NNPS\"]\n verb_tags = [\"VB\", \"VBD\", \"VBG\", \"VBN\", \"VBP\", \"VBZ\"]\n adjective_tags = [\"JJ\", \"JJR\", \"JJS\"]\n \n # check if the POS tag of the given word matches any of the defined tags\n if part_of_speech in noun_tags:\n return \"Noun\"\n elif part_of_speech in verb_tags:\n return \"Verb\"\n elif part_of_speech in adjective_tags:\n return \"Adjective\"\n else:\n return \"Other\"\n\n```\nTo test this function, let's try classifying different words:\n\n```python\n# Testing the classify_words function\nprint(classify_words('cat')) # expected result: Noun\nprint(classify_words('jump')) # expected result: Verb\nprint(classify_words('happy')) # expected result: Adjective\nprint(classify_words('quickly')) # expected result: Adverb\n```\n \nNote that this example solution can correctly identify nouns, verbs, and adjectives, but would not be able to classify other parts of speech such as adverbs correctly as the function is designed to only return one of \"Noun\", \"Verb\", \"Adjective\", or \"Other\"."}]} +{"messages": [{"role": "user", "content": "Compose a pitch for the given business:\nIce Cream Shop"}, {"role": "assistant", "content": "Are you craving a sweet, creamy treat that will satisfy your taste buds and add a smile to your day? Look no further than our Ice Cream Shop! Our premium selection of delicious, hand-crafted ice creams offers something irresistible for everyone. From classic flavors like vanilla and chocolate to exciting new combinations that will tantalize your taste buds, we have everything you need to indulge your sweet tooth.\n\nAt our Ice Cream Shop, we use only the finest and freshest ingredients to craft our signature flavors. Our ice cream is made in small batches to ensure the perfect texture and taste, and we're always experimenting with new recipes to keep things interesting. Whether you prefer traditional flavors or want to try something unique and adventurous, we have you covered.\n\nStop by today and discover the amazing variety of ice cream flavors available at our Ice Cream Shop. You can enjoy your treat in our cozy shop or take it to go, and we guarantee it will be the highlight of your day. Come see why we're the top destination for ice cream lovers in town!"}]} +{"messages": [{"role": "user", "content": "What are some of the best television shows of all time?"}, {"role": "assistant", "content": "There have been many amazing television shows throughout history, and it is difficult to definitively say which ones are the best. However, below are some critically acclaimed shows that have consistently appeared on lists of the greatest TV shows of all time:\n\n1. The Sopranos - This HBO series centers around a New Jersey-based Italian-American mobster, and has often been praised for its complex characters, strong writing, and excellent performances.\n\n2. Breaking Bad - This series tells the story of a chemistry teacher who turns to manufacturing and selling methamphetamine after being diagnosed with cancer. It is widely regarded for its intense storyline, strong acting, and high production values.\n\n3. The Wire - This HBO series is a realistic portrayal of the interconnected lives of criminals, police, and politicians in Baltimore. It's known for its gritty realism and powerful social commentary.\n\n4. Game of Thrones - This fantasy epic focuses on the battle for control of the Seven Kingdoms of Westeros. Game of Thrones is often praised for its engaging characters, intricate plot, and breathtaking visuals.\n\n5. Seinfeld - This show is often called the \"show about nothing.\" It captures the everyday life of a group of friends in New York City. Seinfeld is known for its strong writing, humor, and its influence on modern-day sitcoms.\n\n6. The Simpsons - This animated series follows the misadventures of the Simpson family in the fictional town of Springfield. The Simpsons is celebrated for its humor, social satire, and cultural impact.\n\n7. Friends - This popular sitcom focuses on the lives of six friends living in New York City. Friends is known for its strong ensemble cast, relatable storylines, and memorable humor.\n\n8. Mad Men - This series follows the professional and personal lives of advertising executives in the 1960s. Mad Men is praised for its historical accuracy, strong writing, and nuanced characters.\n\n9. The Twilight Zone - This anthology series features standalone episodes that often have a twist ending. The Twilight Zone is known for its thought-provoking stories, social commentary, and influence on modern-day science fiction.\n\n10. Cheers - This show is set in a bar in Boston and follows the lives of its patrons and employees. Cheers is celebrated for its strong ensemble cast, relatable characters, and humor."}]} +{"messages": [{"role": "user", "content": "Give an example of a traditional folk tale."}, {"role": "assistant", "content": "One example of a traditional folk tale is \"The Tortoise and the Hare.\" This tale, which originated from Aesop’s Fables, tells the story of a hare who ridicules a slow-moving tortoise. Tired of the hare’s boasting, the tortoise challenges him to a race. The hare soon leaves the tortoise far behind and, confident of winning, decides to take a nap midway through the race. When the hare awakens however, he finds that the tortoise, teaching him a lesson, has crawled steadily and carefully towards the finish line and has won the race.\n\nThe moral of the story is that slow and steady wins the race. The tale teaches children not to be arrogant and overconfident in their abilities, but to be diligent and persistent in their efforts."}]} +{"messages": [{"role": "user", "content": "Write some example questions for a customer survey about a home appliance."}, {"role": "assistant", "content": "1. On a scale of 1-10, how satisfied are you with the performance of our home appliance?\n2. How easy was the installation process of the appliance?\n3. Has the appliance met your expectations? If not, why?\n4. How often do you use the appliance?\n5. Have you encountered any technical issues while using the appliance? If so, please describe.\n6. Would you recommend this home appliance to a friend or family member?\n7. How does our appliance compare to similar products from other brands?\n8. How can we improve the design or functionality of the appliance in the future?\n9. Did you find the instructions for using the appliance clear and easy to follow?\n10. Are there any additional features you wish the appliance had?"}]}